NEW Use custom list for eagerloaded relations (#10869)

src/ORM/ArrayLib.php

@@ -274,4 +274,21 @@ class ArrayLib
             }
         }
     }
+
+    /**
+     * Similar to shuffle, but retains the existing association between the keys and the values.
+     * Shuffles the array in place.
+     */
+    public static function shuffleAssociative(array &$array): void
+    {
+        $shuffledArray = [];
+        $keys = array_keys($array);
+        shuffle($keys);
+
+        foreach ($keys as $key) {
+            $shuffledArray[$key] = $array[$key];
+        }
+
+        $array = $shuffledArray;
+    }
 }

src/ORM/Connect/MySQLQuery.php

@@ -64,11 +64,20 @@ class MySQLQuery extends Query
                 }
                 yield $data;
             }
-            // Check for the method first since $this->handle is a mixed type
+
-            if (method_exists($this->handle, 'data_seek')) {
+            // Reset so the query can be iterated over again
-                // Reset so the query can be iterated over again
+            $this->rewind();
-                $this->handle->data_seek(0);
+        }
-            }
+    }
+
+    /**
+     * Rewind to the first row
+     */
+    public function rewind(): void
+    {
+        // Check for the method first since $this->handle is a mixed type
+        if (method_exists($this->handle, 'data_seek')) {
+            $this->handle->data_seek(0);
         }
     }
 

src/ORM/Connect/MySQLStatement.php

@@ -117,9 +117,17 @@ class MySQLStatement extends Query
             yield $row;
         }
 
+        // Reset so the query can be iterated over again
+        $this->rewind();
+    }
+
+    /**
+     * Rewind to the first row
+     */
+    public function rewind(): void
+    {
         // Check for the method first since $this->statement isn't strongly typed
         if (method_exists($this->statement, 'data_seek')) {
-            // Reset so the query can be iterated over again
             $this->statement->data_seek(0);
         }
     }

src/ORM/DataList.php

@@ -60,9 +60,22 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
      */
     protected $finalisedQuery;
 
+    /**
+     * A de-duped list of all relation chains to eagerly fetch data for
+     */
+    private array $eagerLoadRelationChains = [];
 
-    private array $eagerLoadRelations = [];
+    /**
+     * A full list of all relations (including partial and complete relation chains)
+     * that we will eagerly fetch data for
+     *
+     * Used to avoid fetching duplicate relations
+     */
+    private array $eagerLoadAllRelations = [];
 
+    /**
+     * Eagerly loaded relational data
+     */
     private array $eagerLoadedData = [];
 
     /**
@@ -930,50 +943,9 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
     {
         // cache $item->ID at the top of this method to reduce calls to ViewableData::__get()
         $itemID = $item->ID;
-        foreach (array_keys($this->eagerLoadedData) as $eagerLoadRelation) {
+        foreach (array_keys($this->eagerLoadedData) as $relation) {
-            list($dataClasses, $relations) = $this->getEagerLoadVariables($eagerLoadRelation);
+            if (array_key_exists($itemID, $this->eagerLoadedData[$relation])) {
-            $dataClass = $dataClasses[count($dataClasses) - 2];
+                $item->setEagerLoadedData($relation, $this->eagerLoadedData[$relation][$itemID][$relation]);
-            $relation = $relations[count($relations) - 1];
-            foreach (array_keys($this->eagerLoadedData[$eagerLoadRelation]) as $eagerLoadID) {
-                $eagerLoadedData = $this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation];
-                if ($dataClass === $dataClasses[0]) {
-                    if ($eagerLoadID === $itemID) {
-                        $item->setEagerLoadedData($relation, $eagerLoadedData);
-                    }
-                } elseif ($dataClass === $dataClasses[1]) {
-                    $relationData = $item->{$relations[1]}();
-                    if ($relationData instanceof DataObject) {
-                        if ($relationData->ID === $eagerLoadID) {
-                            $subItem = $relationData;
-                        } else {
-                            $subItem = null;
-                        }
-                    } else {
-                        $subItem = $item->{$relations[1]}()->find('ID', $eagerLoadID);
-                    }
-                    if ($subItem) {
-                        $subItem->setEagerLoadedData($relations[2], $eagerLoadedData);
-                    }
-                } elseif ($dataClass === $dataClasses[2]) {
-                    $relationData = $item->{$relations[1]}();
-                    if ($relationData instanceof DataObject) {
-                        $list = new ArrayList([$relationData]);
-                    } else {
-                        $list = $relationData;
-                    }
-                    foreach ($list as $subItem) {
-                        $subRelationData = $subItem->{$relations[2]}();
-                        if ($relationData instanceof DataObject) {
-                            $subList = new ArrayList([$subRelationData]);
-                        } else {
-                            $subList = $subRelationData;
-                        }
-                        $subSubItem = $subList->find('ID', $eagerLoadID);
-                        if ($subSubItem) {
-                            $subSubItem->setEagerLoadedData($relations[3], $eagerLoadedData);
-                        }
-                    }
-                }
             }
         }
     }
@@ -1009,7 +981,7 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
      * a cached result, unless the DataQuery underlying this list has been
      * modified
      *
-     * @return SilverStripe\ORM\Connect\Query
+     * @return Query
      * @internal This API may change in minor releases
      */
     protected function getFinalisedQuery()
@@ -1021,53 +993,52 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
         return $this->finalisedQuery;
     }
 
-    private function getEagerLoadVariables(string $eagerLoadRelation): array
+    private function getEagerLoadVariables(string $relationChain, string $relationName, string $parentDataClass): array
     {
         $schema = DataObject::getSchema();
-        $relations = array_merge(['root'], explode('.', $eagerLoadRelation));
+
-        $dataClasses = [$this->dataClass];
+        $hasOneComponent = $schema->hasOneComponent($parentDataClass, $relationName);
-        $hasOneIDField = null;
+        if ($hasOneComponent) {
-        $belongsToIDField = null;
+            return [
-        $hasManyIDField = null;
+                $hasOneComponent,
-        $manyManyLastComponent = null;
+                'has_one',
-        for ($i = 0; $i < count($relations) - 1; $i++) {
+                $relationName . 'ID',
-            $parentDataClass = $dataClasses[$i];
+            ];
-            $relationName = $relations[$i + 1];
-            $hasOneComponent = $schema->hasOneComponent($parentDataClass, $relationName);
-            if ($hasOneComponent) {
-                $dataClasses[] = $hasOneComponent;
-                $hasOneIDField = $relations[$i + 1] . 'ID';
-                continue;
-            }
-            $belongsToComponent = $schema->belongsToComponent($parentDataClass, $relationName);
-            if ($belongsToComponent) {
-                $dataClasses[] = $belongsToComponent;
-                $belongsToIDField = $schema->getRemoteJoinField($parentDataClass, $relationName, 'belongs_to');
-                continue;
-            }
-            $hasManyComponent = $schema->hasManyComponent($parentDataClass, $relationName);
-            if ($hasManyComponent) {
-                $dataClasses[] = $hasManyComponent;
-                $hasManyIDField = $schema->getRemoteJoinField($parentDataClass, $relationName, 'has_many');
-                continue;
-            }
-            // this works for both many_many and belongs_many_many
-            $manyManyComponent = $schema->manyManyComponent($parentDataClass, $relationName);
-            if ($manyManyComponent) {
-                $dataClasses[] = $manyManyComponent['childClass'];
-                $manyManyComponent['extraFields'] = $schema->manyManyExtraFieldsForComponent($parentDataClass, $relationName) ?: [];
-                if (is_a($manyManyComponent['relationClass'], ManyManyThroughList::class, true)) {
-                    $manyManyComponent['joinClass'] = $manyManyComponent['join'];
-                    $manyManyComponent['join'] = $schema->baseDataTable($manyManyComponent['joinClass']);
-                } else {
-                    $manyManyComponent['joinClass'] = null;
-                }
-                $manyManyLastComponent = $manyManyComponent;
-                continue;
-            }
-            throw new InvalidArgumentException("Invalid relation passed to eagerLoad() - $eagerLoadRelation");
         }
-        return [$dataClasses, $relations, $hasOneIDField, $belongsToIDField, $hasManyIDField, $manyManyLastComponent];
+        $belongsToComponent = $schema->belongsToComponent($parentDataClass, $relationName);
+        if ($belongsToComponent) {
+            return [
+                $belongsToComponent,
+                'belongs_to',
+                $schema->getRemoteJoinField($parentDataClass, $relationName, 'belongs_to'),
+            ];
+        }
+        $hasManyComponent = $schema->hasManyComponent($parentDataClass, $relationName);
+        if ($hasManyComponent) {
+            return [
+                $hasManyComponent,
+                'has_many',
+                $schema->getRemoteJoinField($parentDataClass, $relationName, 'has_many'),
+            ];
+        }
+        // this works for both many_many and belongs_many_many
+        $manyManyComponent = $schema->manyManyComponent($parentDataClass, $relationName);
+        if ($manyManyComponent) {
+            $manyManyComponent['extraFields'] = $schema->manyManyExtraFieldsForComponent($parentDataClass, $relationName) ?: [];
+            if (is_a($manyManyComponent['relationClass'], ManyManyThroughList::class, true)) {
+                $manyManyComponent['joinClass'] = $manyManyComponent['join'];
+                $manyManyComponent['join'] = $schema->baseDataTable($manyManyComponent['joinClass']);
+            } else {
+                $manyManyComponent['joinClass'] = null;
+            }
+            return [
+                $manyManyComponent['childClass'],
+                'many_many',
+                $manyManyComponent,
+            ];
+        }
+
+        throw new InvalidArgumentException("Invalid relation passed to eagerLoad() - $relationChain");
     }
 
     private function executeQuery(): Query
@@ -1079,173 +1050,205 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
 
     private function fetchEagerLoadRelations(Query $query): void
     {
-        if (empty($this->eagerLoadRelations)) {
+        if (empty($this->eagerLoadRelationChains)) {
             return;
         }
         $topLevelIDs = $query->column('ID');
         if (empty($topLevelIDs)) {
             return;
         }
-        $prevRelationArray = [];
+
-        foreach ($this->eagerLoadRelations as $eagerLoadRelation) {
+        foreach ($this->eagerLoadRelationChains as $relationChain) {
-            list(
+            $parentDataClass = $this->dataClass();
-                $dataClasses,
+            $parentIDs = $topLevelIDs;
-                $relations,
+            $parentRelationName = '';
-                $hasOneIDField,
+            /** @var Query|array<DataObject|EagerLoadedList> */
-                $belongsToIDField,
+            $parentRelationData = $query;
-                $hasManyIDField,
+            $chainToDate = [];
-                $manyManyLastComponent
+            foreach (explode('.', $relationChain) as $relationName) {
-            ) = $this->getEagerLoadVariables($eagerLoadRelation);
+                $chainToDate[] = $relationName;
-            $parentDataClass = $dataClasses[count($dataClasses) - 2];
+                list(
-            $relationName = $relations[count($relations) - 1];
-            $relationDataClass = $dataClasses[count($dataClasses) - 1];
-            if ($parentDataClass === $this->dataClass()) {
-                // When we're at "the top of a tree of nested relationships", we can just use the IDs from the query
-                // This is important to do when handling multiple eager-loaded relationship trees.
-                $parentIDs = $topLevelIDs;
-            }
-            // has_one
-            if ($hasOneIDField) {
-                list($prevRelationArray, $parentIDs) = $this->fetchEagerLoadHasOne(
-                    $query,
-                    $prevRelationArray,
-                    $hasOneIDField,
                     $relationDataClass,
-                    $eagerLoadRelation,
+                    $relationType,
-                    $relationName,
+                    $relationComponent,
-                    $parentDataClass
+                ) = $this->getEagerLoadVariables($relationChain, $relationName, $parentDataClass);
-                );
+
-            // belongs_to
+                switch ($relationType) {
-            } elseif ($belongsToIDField) {
+                    case 'has_one':
-                list($prevRelationArray, $parentIDs) = $this->fetchEagerLoadBelongsTo(
+                        list($parentRelationData, $parentIDs) = $this->fetchEagerLoadHasOne(
-                    $parentIDs,
+                            $parentRelationData,
-                    $belongsToIDField,
+                            $relationComponent,
-                    $relationDataClass,
+                            $relationDataClass,
-                    $eagerLoadRelation,
+                            implode('.', $chainToDate),
-                    $relationName
+                            $relationName
-                );
+                        );
-            // has_many
+                        break;
-            } elseif ($hasManyIDField) {
+                    case 'belongs_to':
-                list($prevRelationArray, $parentIDs) = $this->fetchEagerLoadHasMany(
+                        list($parentRelationData, $parentIDs) = $this->fetchEagerLoadBelongsTo(
-                    $parentIDs,
+                            $parentRelationData,
-                    $hasManyIDField,
+                            $parentIDs,
-                    $relationDataClass,
+                            $relationComponent,
-                    $eagerLoadRelation,
+                            $relationDataClass,
-                    $relationName
+                            implode('.', $chainToDate),
-                );
+                            $relationName
-            // many_many + belongs_many_many & many_many_through
+                        );
-            } elseif ($manyManyLastComponent) {
+                        break;
-                list($prevRelationArray, $parentIDs) = $this->fetchEagerLoadManyMany(
+                    case 'has_many':
-                    $manyManyLastComponent,
+                        list($parentRelationData, $parentIDs) = $this->fetchEagerLoadHasMany(
-                    $parentIDs,
+                            $parentRelationData,
-                    $relationDataClass,
+                            $parentIDs,
-                    $eagerLoadRelation,
+                            $relationComponent,
-                    $relationName,
+                            $relationDataClass,
-                    $parentDataClass
+                            implode('.', $chainToDate),
-                );
+                            $relationName,
-            } else {
+                            $parentRelationName
-                throw new LogicException('Something went wrong with the eager loading');
+                        );
+                        break;
+                    case 'many_many':
+                        list($parentRelationData, $parentIDs) = $this->fetchEagerLoadManyMany(
+                            $parentRelationData,
+                            $relationComponent,
+                            $parentIDs,
+                            $relationDataClass,
+                            implode('.', $chainToDate),
+                            $relationName,
+                            $parentRelationName,
+                            $parentDataClass
+                        );
+                        break;
+                    default:
+                        throw new LogicException("Unexpected relation type $relationType");
+                }
+                $parentDataClass = $relationDataClass;
+                $parentRelationName = $relationName;
             }
         }
     }
 
     private function fetchEagerLoadHasOne(
-        Query $query,
+        Query|array $parents,
-        array $parentRecords,
         string $hasOneIDField,
         string $relationDataClass,
-        string $eagerLoadRelation,
+        string $relationChain,
-        string $relationName,
+        string $relationName
-        string $parentDataClass
     ): array {
-        $itemArray = [];
+        $fetchedIDs = [];
-        $relationItemIDs = [];
+        $addTo = [];
 
-        // It's a has_one directly on the records in THIS list
+        // Find which IDs to add, and where each fetched should be added to
-        if ($parentDataClass === $this->dataClass()) {
+        foreach ($parents as $parentData) {
-            foreach ($query as $itemData) {
+            if (is_array($parentData)) {
-                $itemArray[] = [
+                // $parentData represents a record in this DataList
-                    'ID' => $itemData['ID'],
+                $hasOneID = $parentData[$hasOneIDField];
-                    $hasOneIDField => $itemData[$hasOneIDField]
+                $fetchedIDs[] = $hasOneID;
-                ];
+                $addTo[$hasOneID] = $parentData['ID'];
-                $relationItemIDs[] = $itemData[$hasOneIDField];
+            } elseif ($parentData instanceof DataObject) {
-            }
+                // $parentData represents another has_one record
-        // It's a has_one on a list we've already eager-loaded
+                $hasOneID = $parentData->$hasOneIDField;
-        } else {
+                $fetchedIDs[] = $hasOneID;
-            foreach ($parentRecords as $itemData) {
+                $addTo[$hasOneID] = $parentData;
-                $itemArray[] = [
+            } elseif ($parentData instanceof EagerLoadedList) {
-                    'ID' => $itemData->ID,
+                // $parentData represents a has_many or many_many relation
-                    $hasOneIDField => $itemData->$hasOneIDField
+                foreach ($parentData->getRows() as $parentRow) {
-                ];
+                    $hasOneID = $parentRow[$hasOneIDField];
-                $relationItemIDs[] = $itemData->$hasOneIDField;
+                    $fetchedIDs[] = $hasOneID;
+                    $addTo[$hasOneID] = ['ID' => $parentRow['ID'], 'list' => $parentData];
+                }
+            } else {
+                throw new LogicException("Invalid parent for eager loading has_one relation $relationName");
             }
         }
-        $relationArray = DataObject::get($relationDataClass)->byIDs($relationItemIDs)->toArray();
+
-        foreach ($itemArray as $itemData) {
+        $fetchedRecords = DataObject::get($relationDataClass)->byIDs($fetchedIDs)->toArray();
-            foreach ($relationArray as $relationItem) {
+
-                $eagerLoadID = $itemData['ID'];
+        // Add each fetched record to the appropriate place
-                if ($relationItem->ID === $itemData[$hasOneIDField]) {
+        foreach ($fetchedRecords as $fetched) {
-                    $this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relationName] = $relationItem;
+            $fetchedID = $fetched->ID;
+            $added = false;
+            foreach ($addTo as $matchID => $addHere) {
+                if ($matchID === $fetchedID) {
+                    if ($addHere instanceof DataObject) {
+                        $addHere->setEagerLoadedData($relationName, $fetched);
+                    } elseif (is_array($addHere)) {
+                        $addHere['list']->addEagerLoadedData($relationName, $addHere['ID'], $fetched);
+                    } else {
+                        $this->eagerLoadedData[$relationChain][$addHere][$relationName] = $fetched;
+                    }
+                    $added = true;
+                    break;
                 }
             }
+            if (!$added) {
+                throw new LogicException("Couldn't find parent for record $fetchedID on has_one relation $relationName");
+            }
         }
-        return [$relationArray, $relationItemIDs];
+
+        return [$fetchedRecords, $fetchedIDs];
     }
 
     private function fetchEagerLoadBelongsTo(
+        Query|array $parents,
         array $parentIDs,
         string $belongsToIDField,
         string $relationDataClass,
-        string $eagerLoadRelation,
+        string $relationChain,
         string $relationName
     ): array {
         // Get ALL of the items for this relation up front, for ALL of the parents
         // Fetched as an array to avoid sporadic additional queries when the DataList is looped directly
-        $relationArray = DataObject::get($relationDataClass)->filter([$belongsToIDField => $parentIDs])->toArray();
+        $fetchedRecords = DataObject::get($relationDataClass)->filter([$belongsToIDField => $parentIDs])->toArray();
-        $relationItemIDs = [];
+        $fetchedIDs = [];
 
-        // Store the children against the correct parent
+
-        foreach ($relationArray as $relationItem) {
+        // Add fetched record to the correct place
-            $relationItemIDs[] = $relationItem->ID;
+        foreach ($fetchedRecords as $fetched) {
-            $eagerLoadID = $relationItem->$belongsToIDField;
+            $fetchedIDs[] = $fetched->ID;
-            $this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relationName] = $relationItem;
+            $parentID = $fetched->$belongsToIDField;
+            $this->addEagerLoadedDataToParent($parents, $parentID, $relationChain, $relationName, $fetched, 'has_one');
         }
 
-        return [$relationArray, $relationItemIDs];
+        return [$fetchedRecords, $fetchedIDs];
     }
 
     private function fetchEagerLoadHasMany(
+        Query|array $parents,
         array $parentIDs,
         string $hasManyIDField,
         string $relationDataClass,
-        string $eagerLoadRelation,
+        string $relationChain,
         string $relationName
     ): array {
         // Get ALL of the items for this relation up front, for ALL of the parents
         // Fetched as an array to avoid sporadic additional queries when the DataList is looped directly
-        $relationArray = DataObject::get($relationDataClass)->filter([$hasManyIDField => $parentIDs])->toArray();
+        $fetchedRows = DataObject::get($relationDataClass)->filter([$hasManyIDField => $parentIDs])->getFinalisedQuery();
-        $relationItemIDs = [];
+        $fetchedIDs = [];
+        $eagerLoadedLists = [];
 
-        // Store the children in an ArrayList against the correct parent
+        // Store the children in an EagerLoadedList against the correct parent
-        foreach ($relationArray as $relationItem) {
+        foreach ($fetchedRows as $row) {
-            $relationItemIDs[] = $relationItem->ID;
+            $fetchedIDs[] = $row['ID'];
-            $eagerLoadID = $relationItem->$hasManyIDField;
+            $parentID = $row[$hasManyIDField];
-            if (!isset($this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relationName])) {
+            if (isset($eagerLoadedLists[$parentID])) {
-                $arrayList = ArrayList::create();
+                $eagerLoadedList = $eagerLoadedLists[$parentID];
-                $arrayList->setDataClass($relationDataClass);
+            } else {
-                $this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relationName] = $arrayList;
+                // If we haven't created a list yet, create it and add it to the correct parent list/record
+                $eagerLoadedList = EagerLoadedList::create($relationDataClass, HasManyList::class, $parentID);
+                $eagerLoadedLists[$parentID] = $eagerLoadedList;
+                $this->addEagerLoadedDataToParent($parents, $parentID, $relationChain, $relationName, $eagerLoadedList, 'has_many');
             }
-            $this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relationName]->push($relationItem);
+            // Add this row to the list
+            $eagerLoadedList->addRow($row);
         }
 
-        return [$relationArray, $relationItemIDs];
+        return [$eagerLoadedLists, $fetchedIDs];
     }
 
     private function fetchEagerLoadManyMany(
+        Query|array $parents,
         array $manyManyLastComponent,
         array $parentIDs,
         string $relationDataClass,
-        string $eagerLoadRelation,
+        string $relationChain,
         string $relationName,
         string $parentDataClass
     ): array {
@@ -1254,12 +1257,18 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
         $joinTable = $manyManyLastComponent['join'];
         $extraFields = $manyManyLastComponent['extraFields'];
         $joinClass = $manyManyLastComponent['joinClass'];
+        $fetchedRowsArray = [];
+        $fetchedIDs = [];
+        $eagerLoadedLists = [];
 
         // Get the join records so we can correctly identify which children belong to which parents
+        // This also holds extra fields data
         $joinRows = DB::query('SELECT * FROM "' . $joinTable . '" WHERE "' . $parentIDField . '" IN (' . implode(',', $parentIDs) . ')');
 
-        // many_many_through
+        // Use a real RelationList here so that the extraFields and join record are correctly fetched for all relations
+        // There's a lot of special handling for things like DBComposite extra fields, etc.
         if ($joinClass !== null) {
+            // many_many_through
             $relationList = ManyManyThroughList::create(
                 $relationDataClass,
                 $joinClass,
@@ -1268,40 +1277,94 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
                 $extraFields,
                 $relationDataClass,
                 $parentDataClass
-            )->forForeignID($parentIDs);
+            );
-        // many_many + belongs_many_many
         } else {
+            // many_many + belongs_many_many
             $relationList = ManyManyList::create(
                 $relationDataClass,
                 $joinTable,
                 $childIDField,
                 $parentIDField,
                 $extraFields
-            )->forForeignID($parentIDs);
+            );
         }
 
         // Get ALL of the items for this relation up front, for ALL of the parents
-        // Use a real RelationList here so that the extraFields and join record are correctly set for all relations
+        $fetchedRows = $relationList->forForeignID($parentIDs)->getFinalisedQuery();
-        // Fetched as a map so we can get the ID for all records up front (instead of in another nested loop)
-        // Fetched after that as an array because for some reason that performs better in the loop
-        // Note that "Me" is a method on ViewableData that returns $this - i.e. that is the actual DataObject record
-        $relationArray = $relationList->map('ID', 'Me')->toArray();
 
-        // Store the children in an ArrayList against the correct parent
+        foreach ($fetchedRows as $row) {
+            $fetchedRowsArray[$row['ID']] = $row;
+            $fetchedIDs[] = $row['ID'];
+        }
+
+        // Store the children in an EagerLoadedList against the correct parent
         foreach ($joinRows as $row) {
             $parentID = $row[$parentIDField];
             $childID = $row[$childIDField];
-            $relationItem = $relationArray[$childID];
+            $relationItem = $fetchedRowsArray[$childID];
 
-            if (!isset($this->eagerLoadedData[$eagerLoadRelation][$parentID][$relationName])) {
+            if (isset($eagerLoadedLists[$parentID])) {
-                $arrayList = ArrayList::create();
+                $eagerLoadedList = $eagerLoadedLists[$parentID];
-                $arrayList->setDataClass($relationDataClass);
+            } else {
-                $this->eagerLoadedData[$eagerLoadRelation][$parentID][$relationName] = $arrayList;
+                // If we haven't created a list yet, create it and add it to the correct parent list/record
+                $eagerLoadedList = EagerLoadedList::create($relationDataClass, get_class($relationList), $parentID, $manyManyLastComponent);
+                $eagerLoadedLists[$parentID] = $eagerLoadedList;
+                $this->addEagerLoadedDataToParent($parents, $parentID, $relationChain, $relationName, $eagerLoadedList, 'many_many');
             }
-            $this->eagerLoadedData[$eagerLoadRelation][$parentID][$relationName]->push($relationItem);
+            // Add this row to the list
+            $eagerLoadedList->addRow($relationItem);
         }
 
-        return [$relationArray, array_keys($relationArray)];
+        return [$eagerLoadedLists, $fetchedIDs];
+    }
+
+    /**
+     * Adds eager loaded data to the correct parent list or record
+     */
+    private function addEagerLoadedDataToParent(
+        Query|array $parents,
+        int $parentID,
+        string $relationChain,
+        string $relationName,
+        DataObject|EagerLoadedList $eagerLoadedData,
+        string $relationType
+    ): void {
+        $added = false;
+        foreach ($parents as $parentData) {
+            if (is_array($parentData)) {
+                // $parentData represents a record in this DataList
+                if ($parentData['ID'] === $parentID) {
+                    $this->eagerLoadedData[$relationChain][$parentID][$relationName] = $eagerLoadedData;
+                    $added = true;
+                    // Reset the query if we can - but if not, we have to iterate over the whole result set
+                    // so that we will be starting from the beginning again on the next iteration
+                    if (method_exists($parents, 'rewind')) {
+                        $parents->rewind();
+                        break;
+                    }
+                }
+            } elseif ($parentData instanceof DataObject) {
+                // $parentData represents another has_one record
+                if ($parentData->ID === $parentID) {
+                    $parentData->setEagerLoadedData($relationName, $eagerLoadedData);
+                    $added = true;
+                    break;
+                }
+            } elseif ($parentData instanceof EagerLoadedList) {
+                // $parentData represents a has_many or many_many relation
+                if ($parentData->hasID($parentID)) {
+                    $parentData->addEagerLoadedData($relationName, $parentID, $eagerLoadedData);
+                    $added = true;
+                    break;
+                }
+            } else {
+                throw new LogicException("Invalid parent for eager loading $relationType relation $relationName");
+            }
+        }
+
+        if (!$added) {
+            throw new LogicException("Couldn't find parent for $relationType relation $relationName");
+        }
     }
 
     /**
@@ -1310,35 +1373,43 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
      * Eager loading alleviates the N + 1 problem by querying the nested relationship tables before they are
      * needed using a single large `WHERE ID in ($ids)` SQL query instead of many `WHERE RelationID = $id` queries.
      *
-     * You can speicify nested relations by using dot notation, and you can also pass in multiple relations.
+     * You can specify nested relations by using dot notation, and you can also pass in multiple relations.
-     * When speicifying nested relations there is a maximum of 3 levels of relations allowed i.e. 2 dots
+     * When specifying nested relations there is a maximum of 3 levels of relations allowed i.e. 2 dots
      *
      * Example:
      * $myDataList->eagerLoad('MyRelation.NestedRelation.EvenMoreNestedRelation', 'DifferentRelation')
      *
-     * IMPORTANT: Calling eagerLoad() will cause any relations on DataObjects to be returned as an ArrayList
+     * IMPORTANT: Calling eagerLoad() will cause any relations on DataObjects to be returned as an EagerLoadedList
-     * instead of a subclass of DataList such as HasManyList i.e. MyDataObject->MyHasManyRelation() returns an ArrayList
+     * instead of a subclass of DataList such as HasManyList i.e. MyDataObject->MyHasManyRelation() returns an EagerLoadedList
      */
-    public function eagerLoad(...$relations): static
+    public function eagerLoad(...$relationChains): static
     {
-        $arr = [];
+        foreach ($relationChains as $relationChain) {
-        foreach ($relations as $relation) {
+            // Don't add any relations we've added before
-            $parts = explode('.', $relation);
+            if (array_key_exists($relationChain, $this->eagerLoadAllRelations)) {
+                continue;
+            }
+            $parts = explode('.', $relationChain);
             $count = count($parts);
             if ($count > 3) {
-                $message = "Eager loading only supports up to 3 levels of nesting, passed $count levels - $relation";
+                $message = "Eager loading only supports up to 3 levels of nesting, passed $count levels - $relationChain";
                 throw new InvalidArgumentException($message);
             }
-            // Add each relation in the chain as its own entry to be eagerloaded
+            // Remove any smaller parts of chains and only keep the longest chain for each set of relations
-            // e.g. for "Players.Teams.Coaches" you'll have three entries:
+            // e.g. for "Players.Teams.Coaches" we want to make sure to remove these duplicates:
-            // "Players", "Players.Teams", and "Players.Teams.Coaches
+            // "Players" and "Players.Teams"
             $usedParts = [];
             foreach ($parts as $part) {
                 $usedParts[] = $part;
-                $arr[] = implode('.', $usedParts);
+                $item = implode('.', $usedParts);
+                unset($this->eagerLoadRelationChains[$item]);
+                // Keep track of what we've seen before so we don't accidentally add a level 1 relation
+                // (e.g. "Players") to the chains list when we already have it as part of a longer chain
+                // (e.g. "Players.Teams")
+                $this->eagerLoadAllRelations[$item] = $item;
             }
+            $this->eagerLoadRelationChains[$relationChain] = $relationChain;
         }
-        $this->eagerLoadRelations = array_unique(array_merge($this->eagerLoadRelations, $arr));
         return $this;
     }
 

src/ORM/DataObject.php

@@ -1940,8 +1940,10 @@ class DataObject extends ViewableData implements DataObjectInterface, i18nEntity
         return $this;
     }
 
-    public function setEagerLoadedData(string $eagerLoadRelation, mixed $eagerLoadedData): void
+    public function setEagerLoadedData(
-    {
+        string $eagerLoadRelation,
+        EagerLoadedList|DataObject $eagerLoadedData
+    ): void {
         $this->eagerLoadedData[$eagerLoadRelation] = $eagerLoadedData;
     }
 

src/ORM/EagerLoadedList.php

@@ -0,0 +1,1007 @@
+<?php
+
+namespace SilverStripe\ORM;
+
+use SilverStripe\Dev\Debug;
+use SilverStripe\View\ViewableData;
+use SilverStripe\ORM\Connect\DatabaseException;
+use SilverStripe\ORM\FieldType\DBField;
+use BadMethodCallException;
+use InvalidArgumentException;
+use LogicException;
+use Traversable;
+
+/**
+ * Represents an "eager loaded" DataList - i.e. the data has already been fetched from the database
+ * for these records and likely for some of their relations.
+ *
+ * This list is designed to be plug-and-play with the various DataList implementations, with the exception
+ * that because it doesn't make a database query to get its data, some methods are intentionally not implemented.
+ *
+ * Note that when this list represents a relation, adding items to or removing items from this list will NOT
+ * affect the underlying relation in the database. This list is read-only.
+ */
+class EagerLoadedList extends ViewableData implements Relation, SS_List, Filterable, Sortable, Limitable
+{
+    /**
+     * List responsible for instantiating the actual DataObject objects from eager-loaded data
+     */
+    private DataList $dataList;
+
+    /**
+     * Underlying DataObject class for this list
+     */
+    private string $dataClass;
+
+    /**
+     * The ID(s) of the record that owns this list if the list represents a relation
+     * Used for aggregations
+     */
+    private int|array|null $foreignID;
+
+    /**
+     * ID-indexed array that holds the data from SQL queries for the list
+     * @var array<int,array>
+     */
+    private array $rows = [];
+
+    /**
+     * Nested eager-loaded data which applies to relations on records contained in this list
+     * @var array<int,self|DataObject>
+     */
+    private array $eagerLoadedData = [];
+
+    private array $extraFields = [];
+
+    private array $limitOffset = [null, 0];
+
+    private string|array $sort = [];
+
+    /**
+     * Stored here so we can use it when constructing new lists based on this one
+     */
+    private array $manyManyComponent = [];
+
+    public function __construct(string $dataClass, string $dataListClass, int|array|null $foreignID = null, array $manyManyComponent = [])
+    {
+        if (!is_a($dataListClass, DataList::class, true)) {
+            throw new LogicException('$dataListClass must be an instanceof DataList');
+        }
+
+        // relation lists require a valid foreignID or set of IDs
+        if (is_a($dataListClass, RelationList::class, true) && !$this->isValidForeignID($foreignID)) {
+            throw new InvalidArgumentException('$foreignID must be a valid ID for eager loaded relation lists');
+        }
+
+        $this->dataClass = $dataClass;
+        $this->foreignID = $foreignID;
+        $this->manyManyComponent = $manyManyComponent;
+
+        // many_many relation lists have extra constructor args that don't apply for has_many or non-relations
+        if (is_a($dataListClass, ManyManyThroughList::class, true)) {
+            $this->dataList = ManyManyThroughList::create(
+                $dataClass,
+                // If someone instantiates one of these and passes DataObjectSchema::manyManyComponent() directly
+                // the class will be in here as 'join'
+                $manyManyComponent['joinClass'] ?? $manyManyComponent['join'],
+                $manyManyComponent['childField'],
+                $manyManyComponent['parentField'],
+                $manyManyComponent['extraFields'],
+                $dataClass,
+                $manyManyComponent['parentClass']
+            );
+        } elseif (is_a($dataListClass, ManyManyList::class, true)) {
+            $this->dataList = ManyManyList::create(
+                $dataClass,
+                $manyManyComponent['join'],
+                $manyManyComponent['childField'],
+                $manyManyComponent['parentField'],
+                $manyManyComponent['extraFields']
+            );
+        } else {
+            $this->dataList = $dataListClass::create($dataClass, '');
+        }
+
+        if (isset($manyManyComponent['extraFields'])) {
+            $this->extraFields = $manyManyComponent['extraFields'];
+        }
+    }
+
+    /**
+     * Returns true if the variable passed in is valid for use in $this->dataList->forForeignID() on relation lists
+     */
+    private function isValidForeignID(int|array|null $foreignID): bool
+    {
+        // For an array, only return true if the array contains only integers and isn't empty
+        if (is_array($foreignID)) {
+            if (empty($foreignID)) {
+                return false;
+            }
+            foreach ($foreignID as $id) {
+                if (!is_int($id)) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        // ID must be a valid ID int
+        return $foreignID !== null && $foreignID >= 1;
+    }
+
+    /**
+     * Pass in any eager-loaded data which applies to relations on a specific record in this list
+     *
+     * @return $this
+     */
+    public function addEagerLoadedData(string $relation, int $id, self|DataObject $data): static
+    {
+        $this->eagerLoadedData[$id][$relation] = $data;
+        return $this;
+    }
+
+    /**
+     * Get the dataClass name for this list, ie the DataObject ClassName
+     */
+    public function dataClass(): string
+    {
+        return $this->dataClass;
+    }
+
+    public function dbObject($fieldName): ?DBField
+    {
+        return singleton($this->dataClass)->dbObject($fieldName);
+    }
+
+    public function getIDList(): array
+    {
+        $ids = $this->column('ID');
+        return array_combine($ids, $ids);
+    }
+
+    /**
+     * Sets the ComponentSet to be the given ID list
+     * @throws BadMethodCallException
+     */
+    public function setByIDList($idList): void
+    {
+        throw new BadMethodCallException("Can't set the ComponentSet on an EagerLoadedList");
+    }
+
+    /**
+     * Returns a copy of this list with the relationship linked to the given foreign ID
+     * @throws BadMethodCallException
+     */
+    public function forForeignID($id): void
+    {
+        throw new BadMethodCallException("Can't change the foreign ID for an EagerLoadedList");
+    }
+
+    public function getIterator(): Traversable
+    {
+        $limitedRows = $this->getFinalisedRows();
+        foreach ($limitedRows as $row) {
+            yield $this->createDataObject($row);
+        }
+    }
+
+    /**
+     * Get the raw data rows for the records in this list.
+     * Doesn't include nested eagerloaded data.
+     */
+    public function getRows(): array
+    {
+        return array_values($this->rows);
+    }
+
+    public function toArray(): array
+    {
+        $result = [];
+        foreach ($this as $item) {
+            $result[] = $item;
+        }
+        return $result;
+    }
+
+    public function toNestedArray(): array
+    {
+        $result = [];
+        foreach ($this as $item) {
+            $result[] = $item->toMap();
+        }
+        return $result;
+    }
+
+    /**
+     * Add a single row of database data.
+     *
+     * @throws InvalidArgumentException if there is no ID in $row
+     */
+    public function addRow(array $row): static
+    {
+        if (!array_key_exists('ID', $row) || $row['ID'] === null || $row['ID'] === '' || is_array($row['ID'])) {
+            throw new InvalidArgumentException('$row must have a valid ID');
+        }
+        $this->rows[$row['ID']] = $row;
+        return $this;
+    }
+
+    /**
+     * Add multiple rows of database data.
+     *
+     * @throws InvalidArgumentException if any row is missing an ID
+     */
+    public function addRows(array $rows): static
+    {
+        foreach ($rows as $row) {
+            $this->addRow($row);
+        }
+        return $this;
+    }
+
+    /**
+     * Not implemented - use addRow instead.
+     */
+    public function add($item)
+    {
+        throw new BadMethodCallException('Cannot add a DataObject record to EagerLoadedList. Use addRow() to add database rows.');
+    }
+
+    /**
+     * Removes a record from the list. Note that the record will not be removed from the
+     * database - this list is read-only.
+     */
+    public function remove($item): static
+    {
+        $id = $item->ID;
+        if (array_key_exists($id, $this->rows)) {
+            unset($this->rows[$id]);
+        }
+        return $this;
+    }
+
+    public function first(): ?DataObject
+    {
+        $rows = $this->getFinalisedRows();
+        if (count($rows) === 0) {
+            return null;
+        }
+        return $this->byID(array_key_first($rows));
+    }
+
+    public function last(): ?DataObject
+    {
+        $rows = $this->getFinalisedRows();
+        if (count($rows) === 0) {
+            return null;
+        }
+        return $this->byID(array_key_last($rows));
+    }
+
+    public function map($keyField = 'ID', $titleField = 'Title'): Map
+    {
+        return new Map($this, $keyField, $titleField);
+    }
+
+    public function column($colName = 'ID'): array
+    {
+        $rows = $this->getFinalisedRows();
+
+        if (count($rows) === 0) {
+            return [];
+        }
+
+        if ($colName === 'id') {
+            return array_keys($rows);
+        }
+
+        // Don't allow non-existent columns - see DataQuery::column()
+        $id = array_key_first($rows);
+        if (!array_key_exists($colName, $rows[$id])) {
+            throw new InvalidArgumentException('Invalid column name ' . $colName);
+        }
+
+        return array_column($rows, $colName);
+    }
+
+    /**
+     * Returns a unique array of a single field value for all the items in the list
+     *
+     * @param string $colName
+     */
+    public function columnUnique($colName = 'ID'): array
+    {
+        return array_unique($this->column($colName));
+    }
+
+    public function each($callback): static
+    {
+        foreach ($this as $row) {
+            $callback($row);
+        }
+        return $this;
+    }
+
+    public function debug()
+    {
+        // Same implementation as DataList::debug()
+        $val = '<h2>' . static::class . '</h2><ul>';
+        foreach ($this->toNestedArray() as $item) {
+            $val .= '<li style="list-style-type: disc; margin-left: 20px">' . Debug::text($item) . '</li>';
+        }
+        $val .= '</ul>';
+        return $val;
+    }
+
+    /**
+     * Returns whether an item with offset $key exists
+     */
+    public function offsetExists(mixed $key): bool
+    {
+        $count = $this->count();
+        if (!is_int($key) || $count === 0 || $key >= $count) {
+            return false;
+        }
+
+        if ($key < 0) {
+            throw new InvalidArgumentException('$key can not be negative. -1 was provided.');
+        }
+
+        return true;
+    }
+
+    /**
+     * Returns item stored in list with offset $key
+     */
+    public function offsetGet(mixed $key): ?DataObject
+    {
+        if (!is_int($key)) {
+            return null;
+        }
+        return $this->limit(1, $key)->first();
+    }
+
+    /**
+     * Set an item with the key in $key
+     * @throws BadMethodCallException
+     */
+    public function offsetSet(mixed $key, mixed $value): void
+    {
+        // Throw exception for compatability with DataList
+        throw new BadMethodCallException("Can't alter items in an EagerLoadedList using array-access");
+    }
+
+    /**
+     * Unset an item with the key in $key
+     * @throws BadMethodCallException
+     */
+    public function offsetUnset(mixed $key): void
+    {
+        // Throw exception for compatability with DataList
+        throw new BadMethodCallException("Can't alter items in an EagerLoadedList using array-access");
+    }
+
+    public function count(): int
+    {
+        return count($this->getFinalisedRows());
+    }
+
+    /**
+     * Return the maximum value of the given field in this list
+     *
+     * @param string $fieldName
+     */
+    public function max($fieldName): mixed
+    {
+        return max($this->column($fieldName));
+    }
+
+    /**
+     * Return the minimum value of the given field in this list
+     *
+     * @param string $fieldName
+     */
+    public function min($fieldName): mixed
+    {
+        return min($this->column($fieldName));
+    }
+
+    /**
+     * Return the average value of the given field in this list
+     *
+     * @param string $fieldName
+     */
+    public function avg($fieldName): mixed
+    {
+        // We have to rely on the database to either give us the right answer or throw the right exception.
+        // MySQL does wiether things with sum, e.g. an average for the Text field "1_1" is 1 - but
+        // other database implementations could behave differently.
+        $list = $this->foreignID ? $this->dataList->forForeignID($this->foreignID) : $this->dataList;
+        return $list->byIDs($this->column('ID'))->avg($fieldName);
+    }
+
+    /**
+     * Return the sum of the values of the given field in this list
+     *
+     * @param string $fieldName
+     */
+    public function sum($fieldName): int|float
+    {
+        // We have to rely on the database to either give us the right answer or throw the right exception.
+        // MySQL does wiether things with sum, e.g. a sum for the Text field "1_1" is 1 - but
+        // other database implementations could behave differently.
+        $list = $this->foreignID ? $this->dataList->forForeignID($this->foreignID) : $this->dataList;
+        return $list->byIDs($this->column('ID'))->sum($fieldName);
+    }
+
+    /**
+     * Returns true if this list has items
+     */
+    public function exists(): bool
+    {
+        return $this->count() !== 0;
+    }
+
+    public function canFilterBy($fieldName): bool
+    {
+        if (!is_string($fieldName) || empty($this->rows)) {
+            return false;
+        }
+
+        $id = array_key_first($this->rows);
+        return array_key_exists($fieldName, $this->rows[$id]);
+    }
+
+    public function canSortBy($fieldName): bool
+    {
+        return $this->canFilterBy($fieldName);
+    }
+
+    public function find($key, $value): ?DataObject
+    {
+        return $this->filter($key, $value)->first();
+    }
+
+    public function filter(...$args): static
+    {
+        $filters = $this->normaliseFilterArgs($args, __FUNCTION__);
+        $list = clone $this;
+        $list->rows = $this->getMatches($filters);
+        return $list;
+    }
+
+    public function filterAny(...$args): static
+    {
+        $filters = $this->normaliseFilterArgs($args, __FUNCTION__);
+        $list = clone $this;
+        $list->rows = $this->getMatches($filters, true);
+        return $list;
+    }
+
+    public function exclude(...$args): static
+    {
+        $filters = $this->normaliseFilterArgs($args, __FUNCTION__);
+        $toRemove = $this->getMatches($filters);
+        $list = clone $this;
+        foreach ($toRemove as $id => $row) {
+            unset($list->rows[$id]);
+        }
+        return $list;
+    }
+
+    /**
+     * Return a copy of this list which does not contain any items with any of these params
+     */
+    public function excludeAny(...$args): static
+    {
+        $filters = $this->normaliseFilterArgs($args, __FUNCTION__);
+        $toRemove = $this->getMatches($filters, true);
+        $list = clone $this;
+        foreach ($toRemove as $id => $row) {
+            unset($list->rows[$id]);
+        }
+        return $list;
+    }
+
+    /**
+     * Return a new instance of the list with an added filter
+     *
+     * @param array $filterArray
+     */
+    public function addFilter($filterArray): static
+    {
+        $list = clone $this;
+        $list->rows = $this->getMatches($filterArray);
+        return $list;
+    }
+
+    /**
+     * This method returns a copy of this list that does not contain any DataObjects that exists in $list
+     *
+     * The $list passed needs to contain the same dataclass as $this
+     *
+     * @throws InvalidArgumentException
+     */
+    public function subtract(DataList $list): static
+    {
+        if ($this->dataClass() != $list->dataClass()) {
+            throw new InvalidArgumentException('The list passed must have the same dataclass as this class');
+        }
+        return $this->exclude('ID', $list->column('ID'));
+    }
+
+    /**
+     * Validate and process arguments - see DataList::filter(), DataList::exclude(), etc.
+     */
+    private function normaliseFilterArgs(array $arguments, string $function): array
+    {
+        switch (count($arguments)) {
+            case 1:
+                $filter = $arguments[0];
+                break;
+            case 2:
+                $filter = [$arguments[0] => $arguments[1]];
+                break;
+            default:
+                throw new InvalidArgumentException("Incorrect number of arguments passed to $function");
+        }
+        foreach (array_keys($filter) as $column) {
+            if (!$this->canFilterBy($column)) {
+                throw new InvalidArgumentException("Can't filter by column '$column'");
+            }
+        }
+
+        return $filter;
+    }
+
+    /**
+     * Get all rows which match the given filters.
+     * If $any is false, all filters in the $filters array must match.
+     * If $any is true, ANY filter in the $filters array must match.
+     */
+    private function getMatches($filters, bool $any = false): array
+    {
+        $matches = [];
+        foreach ($this->rows as $id => $row) {
+            $doesMatch = true;
+            foreach ($filters as $column => $value) {
+                $extractedValue = $this->extractValue($row, $this->standardiseColumn($column));
+                $strict = $value === null || $extractedValue === null;
+                $doesMatch = $this->doesMatch($column, $value, $extractedValue, $strict);
+                if (!$any && !$doesMatch) {
+                    $doesMatch = false;
+                    break;
+                }
+                if ($any && $doesMatch) {
+                    break;
+                }
+            }
+            if ($doesMatch) {
+                $matches[$id] = $row;
+            }
+        }
+        return $matches;
+    }
+
+    private function doesMatch(string $field, mixed $value1, mixed $value2, bool $strict): bool
+    {
+        if (is_array($value1)) {
+            if (empty($value1)) {
+                // mimics ExactMatchFilter::manyFilter
+                throw new InvalidArgumentException("Cannot filter $field against an empty set");
+            }
+            return in_array($value2, $value1, $strict);
+        }
+
+        if ($strict) {
+            return $value1 === $value2;
+        }
+
+        return $value1 == $value2;
+    }
+
+    /**
+     * Extracts a value from an item in the list, where the item is either an
+     * object or array.
+     *
+     * @param string $key They key for the value to be extracted. Implied mixed type
+     * for compatability with DataList.
+     */
+    private function extractValue(array $row, $key): mixed
+    {
+        if (array_key_exists($key, $row)) {
+            return $row[$key];
+        }
+
+        return null;
+    }
+
+    public function filterByCallback($callback): ArrayList
+    {
+        if (!is_callable($callback)) {
+            throw new LogicException(sprintf(
+                "SS_Filterable::filterByCallback() passed callback must be callable, '%s' given",
+                gettype($callback)
+            ));
+        }
+
+        $output = ArrayList::create();
+        foreach ($this as $item) {
+            if (call_user_func($callback, $item, $this)) {
+                $output->push($item);
+            }
+        }
+        return $output;
+    }
+
+    public function byID($id): ?DataObject
+    {
+        $rows = $this->getFinalisedRows();
+        if (!array_key_exists($id, $rows)) {
+            return null;
+        }
+        return $this->createDataObject($rows[$id]);
+    }
+
+    public function byIDs($ids): static
+    {
+        $list = clone $this;
+        $ids = array_map('intval', (array) $ids);
+        $list->rows = ArrayLib::filter_keys($list->rows, $ids);
+        return $list;
+    }
+
+    public function sort(...$args): static
+    {
+        $count = count($args);
+        if ($count == 0) {
+            return $this;
+        }
+        if ($count > 2) {
+            throw new InvalidArgumentException('This method takes zero, one or two arguments');
+        }
+
+        if ($count == 2) {
+            list($column, $direction) = $args;
+            $sort = [$this->standardiseColumn($column) => $direction];
+        } else {
+            $sort = $args[0];
+            if (!is_string($sort) && !is_array($sort) && !is_null($sort)) {
+                throw new InvalidArgumentException('sort() arguments must either be a string, an array, or null');
+            }
+            if (is_null($sort)) {
+                // Setting sort to null means we just use the default sort order.
+                $list = clone $this;
+                $list->sort = [];
+                return $list;
+            } elseif (empty($sort)) {
+                throw new InvalidArgumentException('Invalid sort parameter');
+            }
+            // If $sort is string then convert string to array to allow for validation
+            if (is_string($sort)) {
+                $newSort = [];
+                // Making the assumption here there are no commas in column names
+                // Other parts of silverstripe will break if there are commas in column names
+                foreach (explode(',', $sort) as $colDir) {
+                    // Using regex instead of explode(' ') in case column name includes spaces
+                    if (preg_match('/^(.+) ([^"]+)$/i', trim($colDir), $matches)) {
+                        list($column, $direction) = [$matches[1], $matches[2]];
+                    } else {
+                        list($column, $direction) = [$colDir, 'ASC'];
+                    }
+                    $newSort[$this->standardiseColumn($column)] = $direction;
+                }
+                $sort = $newSort;
+            }
+        }
+
+        foreach ($sort as $column => $direction) {
+            // validate and normalise sort column
+            $this->validateSortColumn($column);
+
+            // validate sort direction
+            if (!in_array(strtolower($direction), ['asc', 'desc'])) {
+                throw new InvalidArgumentException("Invalid sort direction $direction");
+            }
+        }
+
+        $list = clone $this;
+        $list->sort = $sort;
+        return $list;
+    }
+
+    /**
+     * Shuffle the items in this list
+     */
+    public function shuffle(): static
+    {
+        $list = clone $this;
+        $list->sort = 'shuffle';
+        return $list;
+    }
+
+    private function standardiseColumn(string $column): string
+    {
+        // Strip whitespace and double quotes from single field names e.g. '"Title"'
+        $column = trim($column);
+        if (preg_match('#^"[^"]+"$#', $column)) {
+            $column = str_replace('"', '', $column);
+        }
+        return $column;
+    }
+
+    private function validateSortColumn(string $column): void
+    {
+        $columnName = $column;
+
+        if (preg_match('/^[A-Z0-9\._]+$/i', $column ?? '')) {
+            $relations = explode('.', $column ?? '');
+            $fieldName = array_pop($relations);
+
+            $relationModelClass = $this->dataClass();
+
+            foreach ($relations as $relation) {
+                $prevModelClass = $relationModelClass;
+                /** @var DataObject $singleton */
+                $singleton = singleton($relationModelClass);
+                $relationModelClass = $singleton->getRelationClass($relation);
+                // See DataQuery::applyRelation() which is called indirectly from DataList::validateSortColumn()
+                // for context on these exceptions.
+                if ($relationModelClass === null) {
+                    throw new InvalidArgumentException("$relation is not a relation on model $prevModelClass");
+                }
+                if (in_array($singleton->getRelationType($relation), ['has_many', 'many_many', 'belongs_many_many'])) {
+                    throw new InvalidArgumentException("$relation is not a linear relation on model $prevModelClass");
+                }
+            }
+
+            if (strpos($column, '.') === false) {
+                if (!singleton($relationModelClass)->hasDatabaseField($column)) {
+                    throw new DatabaseException("Unknown column \"$column\"");
+                }
+                $columnName = '"' . $column . '"';
+            } else {
+                // Find the db field the relation belongs to - It will be returned in quoted SQL "TableName"."ColumnName" notation
+                // Note that sqlColumnForField() throws an expected exception if the field doesn't exist on the relation
+                $relationPrefix = DataQuery::applyRelationPrefix($relations);
+                $columnName = DataObject::getSchema()->sqlColumnForField($relationModelClass, $fieldName, $relationPrefix);
+            }
+
+            // All of the above is necessary to ensure the expected exceptions are thrown for invalid relations
+            // But we still need to ultimately throw an exception here, because sorting by relations isn't
+            // currently supported at all for this class.
+            if (!empty($relations)) {
+                throw new InvalidArgumentException('Cannot sort by relations on EagerLoadedList');
+            }
+        }
+
+        // If $columnName is equal to $col it means that it was orginally raw sql or otherwise invalid.
+        if ($columnName === $column) {
+            throw new InvalidArgumentException("Invalid sort column $column");
+        }
+    }
+
+    public function reverse(): static
+    {
+        // No-op if we're gonna shuffle the list anyway
+        if ($this->sort === 'shuffle') {
+            return $this;
+        }
+        // Set the sort order for each clause to be reversed
+        // This is how DataList reverses its list order as well
+        $list = clone $this;
+        foreach ($list->sort as $clause => &$dir) {
+            $dir = (strtoupper($dir) == 'DESC') ? 'ASC' : 'DESC';
+        }
+        return $list;
+    }
+
+    public function limit(?int $length, int $offset = 0): static
+    {
+        if ($length !== null && $length < 0) {
+            throw new InvalidArgumentException("\$length can not be negative. $length was provided.");
+        }
+
+        if ($offset < 0) {
+            throw new InvalidArgumentException("\$offset can not be negative. $offset was provided.");
+        }
+
+        // We don't actually apply the limit immediately, for compatability with the way it works in DataList
+        $list = clone $this;
+        $list->limitOffset = [$length, $offset];
+        return $list;
+    }
+
+    /**
+     * Check if this list has an item with the given ID
+     */
+    public function hasID(int $id): bool
+    {
+        return array_key_exists($id, $this->getFinalisedRows());
+    }
+
+    public function relation($relationName): ?Relation
+    {
+        $ids = $this->column('ID');
+
+        $prototypicalList = null;
+
+        // If we've already got that data loaded, don't trigger a new DB query
+        $relations = [];
+        foreach ($ids as $id) {
+            if (!isset($this->eagerLoadedData[$id][$relationName])) {
+                continue;
+            }
+            $data = $this->eagerLoadedData[$id][$relationName];
+            if (!($data instanceof self)) {
+                // There's no clean way to get the rows back out of DataObject records,
+                // and if it's not a DataObject then we don't know how to handle it,
+                // so fall back to a new DB query
+                break;
+            }
+            $prototypicalList = $data;
+            $relations = array_merge($relations, $data->getRows());
+        }
+
+        if (!empty($relations)) {
+            $relation = EagerLoadedList::create(
+                $prototypicalList->dataClass(),
+                get_class($prototypicalList->dataList),
+                $ids,
+                $prototypicalList->manyManyComponent
+            );
+            $relation->addRows($relations);
+            return $relation;
+        }
+
+        // Trigger a new DB query if needed - see DataList::relation()
+        $singleton = DataObject::singleton($this->dataClass);
+        $relation = $singleton->$relationName($ids);
+
+        return $relation;
+    }
+
+    /**
+     * Create a DataObject from the given SQL row.
+     * At a minimum, $row['ID'] must be set. Unsaved records cannot be eager loaded.
+     *
+     * @param array $row
+     */
+    public function createDataObject($row): DataObject
+    {
+        if (!array_key_exists('ID', $row)) {
+            throw new InvalidArgumentException('$row must have an ID');
+        }
+        $record = $this->dataList->createDataObject($row);
+        $this->setDataObjectEagerLoadedData($row['ID'], $record);
+        return $record;
+    }
+
+    /**
+     * Find the extra field data for a single row of the relationship join
+     * table for many_many relations, given the known child ID.
+     *
+     * @param string $componentName The name of the component (unused, but kept for compatability with ManyManyList)
+     * @param int|string $itemID The ID of the child for the relationship
+     *
+     * @return array Map of fieldName => fieldValue
+     * @throws BadMethodCallException if the relation type for this list is not many_many
+     * @throws InvalidArgumentException if $itemID is not numeric
+     */
+    public function getExtraData($componentName, int|string $itemID): array
+    {
+        if (!in_array(get_class($this->dataList), [ManyManyList::class, ManyManyThroughList::class])) {
+            throw new BadMethodCallException('Cannot have extra fields on this list type');
+        }
+
+        // Allow string IDs for compatability with ManyManyList
+        if (!is_numeric($itemID)) {
+            throw new InvalidArgumentException('$itemID must be an integer or numeric string');
+        }
+
+        $itemID = (int)$itemID;
+        $rows = $this->getFinalisedRows();
+
+        // Skip if no extrafields or record not in this list
+        if (empty($this->extraFields) || !array_key_exists($itemID, $rows)) {
+            return [];
+        }
+
+        $result = [];
+        foreach ($this->extraFields as $fieldName => $spec) {
+            $row = $rows[$itemID];
+            if (array_key_exists($fieldName, $row)) {
+                $result[$fieldName] = $row[$fieldName];
+            } else {
+                $result[$fieldName] = null;
+            }
+        }
+        return $result;
+    }
+
+    /**
+     * Gets the extra fields included in the relationship.
+     *
+     * @return array a map of field names to types
+     * @throws BadMethodCallException if the relation type for this list is not many_many
+     */
+    public function getExtraFields(): array
+    {
+        if (!in_array(get_class($this->dataList), [ManyManyList::class, ManyManyThroughList::class])) {
+            throw new BadMethodCallException('Cannot have extra fields on this list type');
+        }
+        return $this->extraFields;
+    }
+
+    private function setDataObjectEagerLoadedData(int $id, DataObject $item): void
+    {
+        if (array_key_exists($id, $this->eagerLoadedData)) {
+            foreach ($this->eagerLoadedData[$id] as $relation => $data) {
+                $item->setEagerLoadedData($relation, $data);
+            }
+        }
+    }
+
+    /**
+     * Gets the final rows for this list after applying all transformations.
+     * Currently only limit is applied lazily, but others could be done this was as well.
+     */
+    private function getFinalisedRows(): array
+    {
+        return $this->doLimit($this->doSort($this->rows));
+    }
+
+    private function doLimit(array $rows): array
+    {
+        list($length, $offset) = $this->limitOffset;
+
+        // If the limit is 0, return an empty list.
+        if ($length === 0) {
+            return [];
+        }
+
+        return array_slice($rows, $offset, $length, true);
+    }
+
+    private function doSort(array $rows): array
+    {
+        // Do nothing if there's no defined sort order.
+        if (empty($this->sort)) {
+            return $rows;
+        }
+
+        if ($this->sort === 'shuffle') {
+            ArrayLib::shuffleAssociative($rows);
+            return $rows;
+        }
+
+        uasort($rows, function (array $row, array $other): int {
+            $compared = 0;
+            foreach ($this->sort as $column => $direction) {
+                $rowValue = $this->extractValue($row, $column);
+                $otherValue = $this->extractValue($other, $column);
+                // We need to treat numbers differently than numeric strings to match database behaviour
+                if ($this->isNumericNotString($rowValue) && $this->isNumericNotString($otherValue)) {
+                    $compared = $rowValue <=> $otherValue;
+                } else {
+                    $compared = strcasecmp($rowValue ?? '', $otherValue ?? '');
+                }
+                if ($compared !== 0) {
+                    // Reverse the direction for desc; i.e. -1 becomes 1 and 1 becomes -1
+                    if (strtolower($direction) === 'desc') {
+                        $compared *= -1;
+                    }
+                    // If the comparison clearly marks an order, we don't need to check the remaining columns.
+                    break;
+                }
+            }
+            return $compared;
+        });
+
+        return $rows;
+    }
+
+    private function isNumericNotString(mixed $value): bool
+    {
+        return is_numeric($value) && !is_string($value);
+    }
+}

tests/php/ORM/ArrayLibTest.php

@@ -2,6 +2,7 @@
 
 namespace SilverStripe\ORM\Tests;
 
+use PHPUnit\Framework\ExpectationFailedException;
 use SilverStripe\ORM\ArrayLib;
 use SilverStripe\Dev\SapphireTest;
 
@@ -335,4 +336,36 @@ class ArrayLibTest extends SapphireTest
             'New items are iterated over'
         );
     }
+
+    public function testShuffleAssociative()
+    {
+        $list = ['a' => 1, 'b' => 2, 'c' => 3, 'd' => 4, 'e' => 5, 'f' => 6];
+        $copy = $list;
+        // Try shuffling 3 times - it's technically possible the result of a shuffle could be
+        // the exact same order as the original list.
+        for ($attempts = 1; $attempts <= 3; $attempts++) {
+            ArrayLib::shuffleAssociative($copy);
+            // Check value/key association is retained
+            foreach ($list as $key => $value) {
+                $this->assertEquals($value, $copy[$key]);
+            }
+
+            $failed = false;
+            try {
+                // Check the order is different
+                $this->assertNotSame($list, $copy);
+            } catch (ExpectationFailedException $e) {
+                $failed = true;
+                // Only fail the test if we've tried and failed 3 times.
+                if ($attempts === 3) {
+                    throw $e;
+                }
+            }
+
+            // If we've passed the shuffle test, don't retry.
+            if (!$failed) {
+                break;
+            }
+        }
+    }
 }

tests/php/ORM/DataListEagerLoadingTest.php

@@ -4,10 +4,10 @@ namespace SilverStripe\ORM\Tests;
 
 use InvalidArgumentException;
 use SilverStripe\Dev\SapphireTest;
-use SilverStripe\ORM\ArrayList;
 use SilverStripe\ORM\DataList;
 use SilverStripe\ORM\DataObject;
 use SilverStripe\ORM\DB;
+use SilverStripe\ORM\EagerLoadedList;
 use SilverStripe\ORM\ManyManyThroughList;
 use SilverStripe\ORM\Tests\DataListTest\EagerLoading\EagerLoadObject;
 use SilverStripe\ORM\Tests\DataListTest\EagerLoading\HasOneEagerLoadObject;
@@ -287,7 +287,7 @@ class DataListEagerLoadingTest extends SapphireTest
                     'BelongsManyManyEagerLoadObjects.BelongsManyManySubEagerLoadObjects',
                     'MixedManyManyEagerLoadObjects.MixedHasManyEagerLoadObjects.MixedHasOneEagerLoadObject',
                 ],
-                'expected' => 78
+                'expected' => 73
             ],
             [
                 'iden' => 'all',
@@ -1123,7 +1123,7 @@ class DataListEagerLoadingTest extends SapphireTest
         $record->write();
         $record->HasManyEagerLoadObjects()->add(HasManyEagerLoadObject::create(['Title' => 'My related obj']));
         $obj = EagerLoadObject::get()->eagerLoad('HasManyEagerLoadObjects')->first();
-        $this->assertInstanceOf(ArrayList::class, $obj->HasManyEagerLoadObjects());
+        $this->assertInstanceOf(EagerLoadedList::class, $obj->HasManyEagerLoadObjects());
     }
 
     public function testLastHasEagerloadedRelation()
@@ -1132,6 +1132,6 @@ class DataListEagerLoadingTest extends SapphireTest
         $record->write();
         $record->HasManyEagerLoadObjects()->add(HasManyEagerLoadObject::create(['Title' => 'My related obj']));
         $obj = EagerLoadObject::get()->eagerLoad('HasManyEagerLoadObjects')->last();
-        $this->assertInstanceOf(ArrayList::class, $obj->HasManyEagerLoadObjects());
+        $this->assertInstanceOf(EagerLoadedList::class, $obj->HasManyEagerLoadObjects());
     }
 }

tests/php/ORM/DataListTest.php

@@ -2,6 +2,7 @@
 
 namespace SilverStripe\ORM\Tests;
 
+use BadMethodCallException;
 use Exception;
 use InvalidArgumentException;
 use SilverStripe\Core\Convert;
@@ -27,6 +28,11 @@ use SilverStripe\ORM\Tests\DataObjectTest\TeamComment;
 use SilverStripe\ORM\Tests\DataObjectTest\ValidatedObject;
 use SilverStripe\ORM\Tests\ManyManyListTest\Category;
 use SilverStripe\ORM\Connect\DatabaseException;
+use SilverStripe\ORM\FieldType\DBPrimaryKey;
+use SilverStripe\ORM\FieldType\DBText;
+use SilverStripe\ORM\FieldType\DBVarchar;
+use SilverStripe\ORM\Tests\DataObjectTest\RelationChildFirst;
+use SilverStripe\ORM\Tests\DataObjectTest\RelationChildSecond;
 
 class DataListTest extends SapphireTest
 {
@@ -82,6 +88,24 @@ class DataListTest extends SapphireTest
         $this->assertEquals(2, count($list ?? []));
     }
 
+    public function testCount()
+    {
+        $list = new DataList(Team::class);
+        $this->assertSame(6, $list->count());
+
+        $list->removeAll();
+        $this->assertSame(0, $list->count());
+    }
+
+    public function testExists()
+    {
+        $list = new DataList(Team::class);
+        $this->assertTrue($list->exists());
+
+        $list->removeAll();
+        $this->assertFalse($list->exists());
+    }
+
     public function testSubtract()
     {
         $comment1 = $this->objFromFixture(DataObjectTest\TeamComment::class, 'comment1');
@@ -191,6 +215,22 @@ class DataListTest extends SapphireTest
         $this->assertEquals($list, clone($list));
     }
 
+    public function testDbObject()
+    {
+        $list = DataList::create(TeamComment::class);
+        $this->assertInstanceOf(DBPrimaryKey::class, $list->dbObject('ID'));
+        $this->assertInstanceOf(DBVarchar::class, $list->dbObject('Name'));
+        $this->assertInstanceOf(DBText::class, $list->dbObject('Comment'));
+    }
+
+    public function testGetIDList()
+    {
+        $list = DataList::create(TeamComment::class);
+        $idList = $list->getIDList();
+        $this->assertSame($list->column('ID'), array_keys($idList));
+        $this->assertSame($list->column('ID'), array_values($idList));
+    }
+
     public function testSql()
     {
         $db = DB::get_conn();
@@ -456,6 +496,16 @@ class DataListTest extends SapphireTest
         }
     }
 
+    public function testRemove()
+    {
+        $list = Team::get();
+        $obj = $this->objFromFixture(DataObjectTest\Team::class, 'team2');
+
+        $this->assertNotNull($list->byID($obj->ID));
+        $list->remove($obj);
+        $this->assertNull($list->byID($obj->ID));
+    }
+
     /**
      * Test DataList->removeByID()
      */
@@ -2048,6 +2098,42 @@ class DataListTest extends SapphireTest
         $this->assertSQLContains(DB::get_conn()->random() . ' AS "_SortColumn', $list->dataQuery()->sql());
     }
 
+    public function testColumn()
+    {
+        // sorted so postgres won't complain about the order being different
+        $list = RelationChildSecond::get()->sort('Title');
+        $ids = [
+            $this->idFromFixture(RelationChildSecond::class, 'test1'),
+            $this->idFromFixture(RelationChildSecond::class, 'test2'),
+            $this->idFromFixture(RelationChildSecond::class, 'test3'),
+            $this->idFromFixture(RelationChildSecond::class, 'test3-duplicate'),
+        ];
+
+        // Test default
+        $this->assertSame($ids, $list->column());
+
+        // Test specific field
+        $this->assertSame(['Test 1', 'Test 2', 'Test 3', 'Test 3'], $list->column('Title'));
+    }
+
+    public function testColumnUnique()
+    {
+        // sorted so postgres won't complain about the order being different
+        $list = RelationChildSecond::get()->sort('Title');
+        $ids = [
+            $this->idFromFixture(RelationChildSecond::class, 'test1'),
+            $this->idFromFixture(RelationChildSecond::class, 'test2'),
+            $this->idFromFixture(RelationChildSecond::class, 'test3'),
+            $this->idFromFixture(RelationChildSecond::class, 'test3-duplicate'),
+        ];
+
+        // Test default
+        $this->assertSame($ids, $list->columnUnique());
+
+        // Test specific field
+        $this->assertSame(['Test 1', 'Test 2', 'Test 3'], $list->columnUnique('Title'));
+    }
+
     public function testColumnFailureInvalidColumn()
     {
         $this->expectException(InvalidArgumentException::class);
@@ -2103,6 +2189,181 @@ class DataListTest extends SapphireTest
         $this->assertSame('John', $list->last()->Name);
     }
 
+    public function testOffsetGet()
+    {
+        $list = TeamComment::get()->sort('Name');
+        $this->assertEquals('Bob', $list->offsetGet(0)->Name);
+        $this->assertEquals('Joe', $list->offsetGet(1)->Name);
+        $this->assertEquals('Phil', $list->offsetGet(2)->Name);
+        $this->assertNull($list->offsetGet(999));
+    }
+
+    public function testOffsetExists()
+    {
+        $list = TeamComment::get()->sort('Name');
+        $this->assertTrue($list->offsetExists(0));
+        $this->assertTrue($list->offsetExists(1));
+        $this->assertTrue($list->offsetExists(2));
+        $this->assertFalse($list->offsetExists(999));
+    }
+
+    public function testOffsetGetNegative()
+    {
+        $list = TeamComment::get()->sort('Name');
+        $this->expectException(InvalidArgumentException::class);
+        $this->expectExceptionMessage('$offset can not be negative. -1 was provided.');
+        $list->offsetGet(-1);
+    }
+
+    public function testOffsetExistsNegative()
+    {
+        $list = TeamComment::get()->sort('Name');
+        $this->expectException(InvalidArgumentException::class);
+        $this->expectExceptionMessage('$offset can not be negative. -1 was provided.');
+        $list->offsetExists(-1);
+    }
+
+    public function testOffsetSet()
+    {
+        $list = TeamComment::get()->sort('Name');
+        $this->expectException(BadMethodCallException::class);
+        $this->expectExceptionMessage("Can't alter items in a DataList using array-access");
+        $list->offsetSet(0, null);
+    }
+
+    public function testOffsetUnset()
+    {
+        $list = TeamComment::get()->sort('Name');
+        $this->expectException(BadMethodCallException::class);
+        $this->expectExceptionMessage("Can't alter items in a DataList using array-access");
+        $list->offsetUnset(0);
+    }
+
+    /**
+     * @dataProvider provideRelation
+     */
+    public function testRelation(string $parentClass, string $relation, ?array $expected)
+    {
+        $list = $parentClass::get()->relation($relation);
+        if ($expected === null) {
+            $this->assertNull($list);
+        } else {
+            $this->assertListEquals($expected, $list);
+        }
+    }
+
+    public function provideRelation()
+    {
+        return [
+            'many_many' => [
+                'parentClass' => RelationChildFirst::class,
+                'relation' => 'ManyNext',
+                'expected' => [
+                    ['Title' => 'Test 1'],
+                    ['Title' => 'Test 2'],
+                    ['Title' => 'Test 3'],
+                ],
+            ],
+            'has_many' => [
+                'parentClass' => Team::class,
+                'relation' => 'SubTeams',
+                'expected' => [
+                    ['Title' => 'Subteam 1'],
+                ],
+            ],
+            // calling relation() for a has_one just gives you null
+            'has_one' => [
+                'parentClass' => DataObjectTest\Company::class,
+                'relation' => 'Owner',
+                'expected' => null,
+            ],
+        ];
+    }
+
+    /**
+     * @dataProvider provideCreateDataObject
+     */
+    public function testCreateDataObject(string $dataClass, string $realClass, array $row)
+    {
+        $list = new DataList($dataClass);
+        $obj = $list->createDataObject($row);
+
+        // Validate the class is correct
+        $this->assertSame($realClass, get_class($obj));
+
+        // Validates all fields are available
+        foreach ($row as $field => $value) {
+            $this->assertSame($value, $obj->$field);
+        }
+
+        // Validates hydration only used if the row has an ID
+        if (array_key_exists('ID', $row)) {
+            $this->assertFalse($obj->isChanged());
+        } else {
+            $this->assertTrue($obj->isChanged());
+        }
+    }
+
+    public function provideCreateDataObject()
+    {
+        return [
+            'no ClassName' => [
+                'dataClass' => Team::class,
+                'realClass' => Team::class,
+                'row' => [
+                    'ID' => 1,
+                    'Title' => 'Team 1',
+                    'NumericField' => '1',
+                    // Extra field that doesn't exist on that class
+                    'SubclassDatabaseField' => 'this shouldnt be there',
+                ],
+            ],
+            'subclassed ClassName' => [
+                'dataClass' => Team::class,
+                'realClass' => SubTeam::class,
+                'row' => [
+                    'ClassName' => SubTeam::class,
+                    'ID' => 1,
+                    'Title' => 'Team 1',
+                    'SubclassDatabaseField' => 'this time it should be there',
+                ],
+            ],
+            'RecordClassName takes precedence' => [
+                'dataClass' => Team::class,
+                'realClass' => SubTeam::class,
+                'row' => [
+                    'ClassName' => Player::class,
+                    'RecordClassName' => SubTeam::class,
+                    'ID' => 1,
+                    'Title' => 'Team 1',
+                    'SubclassDatabaseField' => 'this time it should be there',
+                ],
+            ],
+            'No ID' => [
+                'dataClass' => Team::class,
+                'realClass' => Team::class,
+                'row' => [
+                    'Title' => 'Team 1',
+                    'NumericField' => '1',
+                    'SubclassDatabaseField' => 'this shouldnt be there',
+                ],
+            ],
+        ];
+    }
+
+    public function testDebug()
+    {
+        $list = Sortable::get();
+
+        $result = $list->debug();
+        $this->assertStringStartsWith('<h2>' . DataList::class . '</h2>', $result);
+        $this->assertMatchesRegularExpression(
+            '/<ul>\s*(<li style="list-style-type: disc; margin-left: 20px">.*?<\/li>)+\s*<\/ul>/s',
+            $result
+        );
+        $this->assertStringEndsWith('</ul>', $result);
+    }
+
     public function testChunkedFetch()
     {
         $expectedIDs = Team::get()->map('ID', 'ID')->toArray();

tests/php/ORM/DataObjectTest.php

@@ -1304,6 +1304,7 @@ class DataObjectTest extends SapphireTest
             'LastEdited',
             'Created',
             'Title',
+            'NumericField',
             'DatabaseField',
             'ExtendedDatabaseField',
             'CaptainID',
@@ -1327,6 +1328,7 @@ class DataObjectTest extends SapphireTest
             'LastEdited',
             'Created',
             'Title',
+            'NumericField',
             'DatabaseField',
             'ExtendedDatabaseField',
             'CaptainID',
@@ -1350,6 +1352,7 @@ class DataObjectTest extends SapphireTest
             'LastEdited',
             'Created',
             'Title',
+            'NumericField',
             'DatabaseField',
             'ExtendedDatabaseField',
             'CaptainID',

tests/php/ORM/DataObjectTest.yml

@@ -31,12 +31,14 @@ SilverStripe\ORM\Tests\DataObjectTest\SubEquipmentCompany:
 SilverStripe\ORM\Tests\DataObjectTest\Team:
   team1:
     Title: Team 1
+    NumericField: 2
     Sponsors:
       - =>SilverStripe\ORM\Tests\DataObjectTest\EquipmentCompany.equipmentcompany1
       - =>SilverStripe\ORM\Tests\DataObjectTest\EquipmentCompany.equipmentcompany2
     EquipmentSuppliers: =>SilverStripe\ORM\Tests\DataObjectTest\EquipmentCompany.equipmentcompany2
   team2:
     Title: Team 2
+    NumericField: 20
     Sponsors:
       - =>SilverStripe\ORM\Tests\DataObjectTest\EquipmentCompany.equipmentcompany2
       - =>SilverStripe\ORM\Tests\DataObjectTest\SubEquipmentCompany.subequipmentcompany1
@@ -45,6 +47,7 @@ SilverStripe\ORM\Tests\DataObjectTest\Team:
       - =>SilverStripe\ORM\Tests\DataObjectTest\EquipmentCompany.equipmentcompany2
   team3:
     Title: Team 3
+    NumericField: 5
 SilverStripe\ORM\Tests\DataObjectTest\Player:
   captain1:
     FirstName: Captain
@@ -69,6 +72,7 @@ SilverStripe\ORM\Tests\DataObjectTest\Player:
 SilverStripe\ORM\Tests\DataObjectTest\SubTeam:
   subteam1:
     Title: Subteam 1
+    NumericField: 7
     SubclassDatabaseField: Subclassed 1
     SubclassFieldWithOverride: DB value of SubclassFieldWithOverride
     ExtendedDatabaseField: Extended 1
@@ -169,6 +173,8 @@ SilverStripe\ORM\Tests\DataObjectTest\RelationChildSecond:
     Title: 'Test 2'
   test3:
     Title: 'Test 3'
+  test3-duplicate:
+    Title: 'Test 3'
 SilverStripe\ORM\Tests\DataObjectTest\RelationChildFirst:
   test1:
     Title: 'Test1'

tests/php/ORM/DataObjectTest/Team.php

@@ -31,6 +31,7 @@ class Team extends DataObject implements TestOnly
     private static $db = [
         'Title' => 'Varchar',
         'DatabaseField' => 'HTMLVarchar',
+        'NumericField' => 'Int',
     ];
 
     private static $has_one = [

tests/php/ORM/DatabaseTest.php

@@ -413,4 +413,85 @@ class DatabaseTest extends SapphireTest
             $i++;
         }
     }
+
+    public function testRewind()
+    {
+        $inputData = ['one', 'two', 'three', 'four'];
+
+        foreach ($inputData as $i => $text) {
+            $x = new MyObject();
+            $x->MyField = $text;
+            $x->MyInt = $i;
+            $x->write();
+        }
+
+        $query = DB::query('SELECT "MyInt", "MyField" FROM "DatabaseTest_MyObject" ORDER BY "MyInt"');
+
+        if (!method_exists($query, 'rewind')) {
+            $class = get_class($query);
+            $this->markTestSkipped("Query subclass $class doesn't implement rewind()");
+        }
+
+        $i = 0;
+        foreach ($query as $record) {
+            $this->assertEquals($inputData[$i], $record['MyField']);
+            $i++;
+            if ($i > 1) {
+                break;
+            }
+        }
+
+        $query->rewind();
+
+        // Start again from the beginning since we called rewind
+        $i = 0;
+        foreach ($query as $record) {
+            $this->assertEquals($inputData[$i], $record['MyField']);
+            $i++;
+        }
+    }
+
+    public function testRewindWithPredicates()
+    {
+        $inputData = ['one', 'two', 'three', 'four'];
+
+        foreach ($inputData as $i => $text) {
+            $x = new MyObject();
+            $x->MyField = $text;
+            $x->MyInt = $i;
+            $x->write();
+        }
+
+        // Note that by including a WHERE statement with predicates
+        // with MySQL the result is in a MySQLStatement object rather than a MySQLQuery object.
+        $query = SQLSelect::create(
+            ['"MyInt"', '"MyField"'],
+            '"DatabaseTest_MyObject"',
+            ['MyInt IN (?,?,?,?,?)' => [0,1,2,3,4]],
+            ['"MyInt"']
+        )->execute();
+
+        if (!method_exists($query, 'rewind')) {
+            $class = get_class($query);
+            $this->markTestSkipped("Query subclass $class doesn't implement rewind()");
+        }
+
+        $i = 0;
+        foreach ($query as $record) {
+            $this->assertEquals($inputData[$i], $record['MyField']);
+            $i++;
+            if ($i > 1) {
+                break;
+            }
+        }
+
+        $query->rewind();
+
+        // Start again from the beginning since we called rewind
+        $i = 0;
+        foreach ($query as $record) {
+            $this->assertEquals($inputData[$i], $record['MyField']);
+            $i++;
+        }
+    }
 }

tests/php/ORM/EagerLoadedListTest.php

@@ -0,0 +1,1885 @@
+<?php
+
+namespace SilverStripe\ORM\Tests;
+
+use BadMethodCallException;
+use InvalidArgumentException;
+use PHPUnit\Framework\ExpectationFailedException;
+use SilverStripe\Core\Injector\Injector;
+use SilverStripe\Dev\SapphireTest;
+use SilverStripe\ORM\Connect\MySQLiConnector;
+use SilverStripe\ORM\EagerLoadedList;
+use SilverStripe\ORM\DB;
+use SilverStripe\ORM\Filterable;
+use SilverStripe\ORM\Tests\DataObjectTest\EquipmentCompany;
+use SilverStripe\ORM\Tests\DataObjectTest\Fan;
+use SilverStripe\ORM\Tests\DataObjectTest\Player;
+use SilverStripe\ORM\Tests\DataObjectTest\Sortable;
+use SilverStripe\ORM\Tests\DataObjectTest\SubTeam;
+use SilverStripe\ORM\Tests\DataObjectTest\Team;
+use SilverStripe\ORM\Tests\DataObjectTest\TeamComment;
+use SilverStripe\ORM\Tests\DataObjectTest\ValidatedObject;
+use SilverStripe\ORM\Tests\ManyManyListTest\Category;
+use SilverStripe\ORM\Connect\DatabaseException;
+use SilverStripe\ORM\DataList;
+use SilverStripe\ORM\DataObject;
+use SilverStripe\ORM\FieldType\DBPrimaryKey;
+use SilverStripe\ORM\FieldType\DBText;
+use SilverStripe\ORM\FieldType\DBVarchar;
+use SilverStripe\ORM\HasManyList;
+use SilverStripe\ORM\ManyManyList;
+use SilverStripe\ORM\ManyManyThroughList;
+use SilverStripe\ORM\Tests\DataObjectTest\RelationChildFirst;
+use SilverStripe\ORM\Tests\DataObjectTest\RelationChildSecond;
+
+class EagerLoadedListTest extends SapphireTest
+{
+    // Borrow the model from DataObjectTest
+    protected static $fixture_file = 'DataObjectTest.yml';
+
+    public static function getExtraDataObjects()
+    {
+        return DataListTest::getExtraDataObjects();
+    }
+
+    private function getBasicRecordRows(): array
+    {
+        return [
+            [
+                'ID' => 1,
+                'Name' => 'test obj 1',
+                'Created' => '2013-01-01 00:00:00',
+            ],
+            [
+                'ID' => 2,
+                'Name' => 'test obj 2',
+                'Created' => '2023-01-01 00:00:00',
+            ],
+            [
+                'ID' => 3,
+                'Name' => 'test obj 3',
+                'Created' => '2023-01-01 00:00:00',
+            ],
+        ];
+    }
+
+    private function getListWithRecords(
+        string|DataList $data,
+        string $dataListClass = DataList::class,
+        ?int $foreignID = null,
+        ?array $manyManyComponentData = null
+    ): EagerLoadedList {
+        // Get some garbage values for the manymany component so we don't get errors
+        // If the component is actually needed, it'll be passed in
+        if ($manyManyComponentData === null) {
+            $manyManyComponent = [];
+            if (in_array($dataListClass, [ManyManyThroughList::class, ManyManyList::class])) {
+                $manyManyComponent['join'] = DataObject::class;
+                $manyManyComponent['childField'] = '';
+                $manyManyComponent['parentField'] = '';
+                $manyManyComponent['parentClass'] = DataObject::class;
+                $manyManyComponent['extraFields'] = [];
+            }
+        } else {
+            list($parentClass, $relationName) = $manyManyComponentData;
+            $manyManyComponent = DataObject::getSchema()->manyManyComponent($parentClass, $relationName);
+            $manyManyComponent['extraFields'] = DataObject::getSchema()->manyManyExtraFieldsForComponent($parentClass, $relationName);
+        }
+
+        if ($data instanceof DataList) {
+            $dataClass = $data->dataClass();
+            $query = $data;
+        } else {
+            $dataClass = $data;
+            $query = DataObject::get($dataClass);
+        }
+        if ($foreignID === null && $dataListClass !== DataList::class) {
+            $foreignID = 9999;
+        }
+        $list = new EagerLoadedList($dataClass, $dataListClass, $foreignID, $manyManyComponent);
+        foreach ($query->dataQuery()->execute() as $row) {
+            $list->addRow($row);
+        }
+        return $list;
+    }
+
+    public function testHasID()
+    {
+        $list = new EagerLoadedList(Sortable::class, DataList::class);
+        foreach ($this->getBasicRecordRows() as $row) {
+            $list->addRow($row);
+        }
+        $this->assertTrue($list->hasID(3));
+        $this->assertFalse($list->hasID(999));
+    }
+
+    public function testDataClass()
+    {
+        $dataClass = TeamComment::class;
+        $list = new EagerLoadedList($dataClass, DataList::class);
+        $this->assertEquals(TeamComment::class, $list->dataClass());
+    }
+
+    public function testDataClassCaseInsensitive()
+    {
+        $dataClass = strtolower(TeamComment::class);
+        $list = new EagerLoadedList($dataClass, DataList::class);
+        $list->addRow(['ID' => 1]);
+        $this->assertInstanceOf($dataClass, $list->first());
+    }
+
+    public function testClone()
+    {
+        $list = new EagerLoadedList(ValidatedObject::class, DataList::class);
+        $list->addRow(['ID' => 1]);
+        $clone = clone($list);
+
+        $this->assertEquals($list, $clone);
+        $this->assertEquals($list->column(), $clone->column());
+
+        $clone->addRow(['ID' => 2]);
+        $this->assertNotEquals($list->column(), $clone->column());
+    }
+
+    public function testDbObject()
+    {
+        $list = new EagerLoadedList(TeamComment::class, DataList::class);
+        $this->assertInstanceOf(DBPrimaryKey::class, $list->dbObject('ID'));
+        $this->assertInstanceOf(DBVarchar::class, $list->dbObject('Name'));
+        $this->assertInstanceOf(DBText::class, $list->dbObject('Comment'));
+    }
+
+    public function testGetIDList()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $idList = $list->getIDList();
+        $this->assertSame($list->column('ID'), array_keys($idList));
+        $this->assertSame($list->column('ID'), array_values($idList));
+    }
+
+    public function testSetByIDList()
+    {
+        $list = new EagerLoadedList(TeamComment::class, DataList::class);
+        $this->expectException(BadMethodCallException::class);
+        $this->expectExceptionMessage("Can't set the ComponentSet on an EagerLoadedList");
+        $list->setByIDList([1,2,3]);
+    }
+
+    public function testForForeignID()
+    {
+        $list = new EagerLoadedList(TeamComment::class, DataList::class);
+        $this->expectException(BadMethodCallException::class);
+        $this->expectExceptionMessage("Can't change the foreign ID for an EagerLoadedList");
+        $list->forForeignID(1);
+    }
+
+    /**
+     * Also tests addRows at the same time
+     */
+    public function testGetRows()
+    {
+        $list = new EagerLoadedList(TeamComment::class, DataList::class);
+        $rows = [
+            [
+                'ID' => 202,
+                'Name' => 'Wobuffet',
+            ],
+            [
+                'ID' => 25,
+                'Name' => 'Pikachu',
+            ]
+        ];
+        $list->addRows($rows);
+        $this->assertSame($rows, $list->getRows());
+
+        // Check we can still add them on afterward
+        $newRow = [
+            'ID' => 1,
+            'Name' => 'Bulbasaur'
+        ];
+        $rows[] = $newRow;
+        $list->addRows([$newRow]);
+        $this->assertSame($rows, $list->getRows());
+    }
+
+    /**
+     * @dataProvider provideAddRowBadID
+     */
+    public function testAddRowBadID(array $row)
+    {
+        $list = new EagerLoadedList(TeamComment::class, DataList::class);
+        $this->expectException(InvalidArgumentException::class);
+        $this->expectExceptionMessage('$row must have a valid ID');
+        $list->addRow($row);
+    }
+
+    public function provideAddRowBadID()
+    {
+        return [
+            [['ID' => null]],
+            [['ID' => '']],
+            [['ID' => [1,2,3]]],
+            [['Name' => 'No ID provided']],
+        ];
+    }
+
+    public function testCount()
+    {
+        $list = new EagerLoadedList(Team::class, DataList::class);
+        $this->assertSame(0, $list->count());
+
+        $list->addRows([
+            ['ID' => 1],
+            ['ID' => 2],
+            ['ID' => 3],
+            ['ID' => 4],
+        ]);
+        $this->assertSame(4, $list->count());
+    }
+
+    public function testExists()
+    {
+        $list = new EagerLoadedList(Team::class, DataList::class);
+        $this->assertFalse($list->exists());
+
+        $list->addRows([
+            ['ID' => 1],
+            ['ID' => 2],
+            ['ID' => 3],
+            ['ID' => 4],
+        ]);
+        $this->assertTrue($list->exists());
+    }
+
+    /**
+     * @dataProvider provideIteration
+     */
+    public function testIteration(string $dataListClass): void
+    {
+        // Get some garbage values for the manymany component so we don't get errors.
+        // Real relations aren't necessary for this test.
+        $manyManyComponent = [];
+        if (in_array($dataListClass, [ManyManyThroughList::class, ManyManyList::class])) {
+            $manyManyComponent['join'] = DataObject::class;
+            $manyManyComponent['childField'] = '';
+            $manyManyComponent['parentField'] = '';
+            $manyManyComponent['parentClass'] = DataObject::class;
+            $manyManyComponent['extraFields'] = [];
+        }
+
+        $rows = $this->getBasicRecordRows();
+        $eagerloadedDataClass = Sortable::class;
+
+        $foreignID = $dataListClass === DataList::class ? null : 9999;
+        $list = new EagerLoadedList($eagerloadedDataClass, $dataListClass, $foreignID, $manyManyComponent);
+        foreach ($rows as $row) {
+            $list->addRow($row);
+        }
+
+        // Validate that the list has the correct records with all the right values
+        $this->iterate($list, $rows, array_column($rows, 'ID'));
+
+        // Validate a repeated iteration works correctly (this has broken for other lists in the past)
+        $this->iterate($list, $rows, array_column($rows, 'ID'));
+    }
+
+    public function provideIteration()
+    {
+        return [
+            [DataList::class],
+            [HasManyList::class],
+            [ManyManyThroughList::class],
+            [ManyManyList::class],
+        ];
+    }
+
+    private function iterate(EagerLoadedList $list, array $rows, array $expectedIDs): void
+    {
+        $foundIDs = [];
+        foreach ($list as $record) {
+            // Assert the correct class is used for the records
+            $this->assertInstanceOf($list->dataClass(), $record);
+            // Get the row this record is for
+            $matches = array_filter($rows, function ($row) use ($record) {
+                return $row['ID'] === $record->ID;
+            });
+            $row = $matches[array_key_first($matches)];
+            // Assert field values are correct
+            foreach ($row as $field => $value) {
+                $this->assertSame($value, $record->$field);
+            }
+            $foundIDs[] = $record->ID;
+        }
+        // Assert all (and only) the expected records were included in the list
+        $this->assertSame($expectedIDs, $foundIDs);
+    }
+
+    /**
+     * @dataProvider provideFilter
+     */
+    public function testFilter(
+        string $dataListClass,
+        string $eagerloadedDataClass,
+        array $rows,
+        array $filter,
+        array $expectedIDs
+    ): void {
+        // Get some garbage values for the manymany component so we don't get errors.
+        // Real relations aren't necessary for this test.
+        $manyManyComponent = [];
+        if (in_array($dataListClass, [ManyManyThroughList::class, ManyManyList::class])) {
+            $manyManyComponent['join'] = DataObject::class;
+            $manyManyComponent['childField'] = '';
+            $manyManyComponent['parentField'] = '';
+            $manyManyComponent['parentClass'] = DataObject::class;
+            $manyManyComponent['extraFields'] = [];
+        }
+
+        $foreignID = $dataListClass === DataList::class ? null : 9999;
+        $list = new EagerLoadedList($eagerloadedDataClass, $dataListClass, $foreignID, $manyManyComponent);
+        foreach ($rows as $row) {
+            $list->addRow($row);
+        }
+        $filteredList = $list->filter($filter);
+
+        // Validate that the unfiltered list still has all records, and the filtered list has the expected amount
+        $this->assertCount(count($rows), $list);
+        $this->assertCount(count($expectedIDs), $filteredList);
+
+        // Validate that the filtered list has the CORRECT records
+        $this->iterate($list, $rows, array_column($rows, 'ID'));
+    }
+
+    public function provideFilter(): array
+    {
+        $rows = $this->getBasicRecordRows();
+        return [
+            [
+                'dataListClass' => DataList::class,
+                'eagerloadedDataClass' => ValidatedObject::class,
+                $rows,
+                'filter' => ['Created' => '2023-01-01 00:00:00'],
+                'expected' => [2, 3],
+            ],
+            [
+                'dataListClass' => HasManyList::class,
+                'eagerloadedDataClass' => ValidatedObject::class,
+                $rows,
+                'filter' => ['Created' => '2023-01-01 00:00:00'],
+                'expected' => [2, 3],
+            ],
+            [
+                'dataListClass' => ManyManyList::class,
+                'eagerloadedDataClass' => ValidatedObject::class,
+                $rows,
+                'filter' => ['Created' => '2023-12-01 00:00:00'],
+                'expected' => [],
+            ],
+            [
+                'dataListClass' => ManyManyThroughList::class,
+                'eagerloadedDataClass' => ValidatedObject::class,
+                $rows,
+                'filter' => [
+                    'Created' => '2023-01-01 00:00:00',
+                    'Name' => 'test obj 3',
+                ],
+                'expected' => [3],
+            ],
+            [
+                'dataListClass' => ManyManyThroughList::class,
+                'eagerloadedDataClass' => ValidatedObject::class,
+                $rows,
+                'filter' => [
+                    'Created' => '2023-01-01 00:00:00',
+                    'Name' => 'not there',
+                ],
+                'expected' => [],
+            ],
+            [
+                'dataListClass' => ManyManyThroughList::class,
+                'eagerloadedDataClass' => ValidatedObject::class,
+                $rows,
+                'filter' => [
+                    'Name' => ['test obj 1', 'test obj 3', 'not there'],
+                ],
+                'expected' => [1, 3],
+            ],
+            [
+                'dataListClass' => ManyManyThroughList::class,
+                'eagerloadedDataClass' => ValidatedObject::class,
+                $rows,
+                'filter' => [
+                    'Name' => ['not there', 'also not there'],
+                ],
+                'expected' => [],
+            ],
+            [
+                'dataListClass' => ManyManyThroughList::class,
+                'eagerloadedDataClass' => ValidatedObject::class,
+                $rows,
+                // Filter by ID is handled slightly differently than other fields
+                'filter' => [
+                    'ID' => [1, 2],
+                ],
+                'expected' => [1, 2],
+            ],
+        ];
+    }
+
+    public function testFilterByInvalidColumn()
+    {
+        $list = new EagerLoadedList(ValidatedObject::class, DataList::class);
+        $list->addRow(['ID' => 1]);
+        $this->expectException(InvalidArgumentException::class);
+        $this->expectExceptionMessage("Can't filter by column 'NotRealField'");
+        $list->filter(['NotRealField' => 'anything']);
+    }
+
+    public function testFilterByRelationColumn()
+    {
+        $list = new EagerLoadedList(Team::class, DataList::class);
+        $list->addRow(['ID' => 1, 'CaptainID' => 1]);
+        $this->expectException(InvalidArgumentException::class);
+        $this->expectExceptionMessage("Can't filter by column 'Captain.ShirtNumber'");
+        $list->filter(['Captain.ShirtNumber' => 'anything']);
+    }
+
+    /**
+     * @dataProvider provideFilterByWrongNumArgs
+     */
+    public function testFilterByWrongNumArgs(...$args)
+    {
+        $list = new EagerLoadedList(ValidatedObject::class, DataList::class);
+        $list->addRow(['ID' => 1]);
+        $this->expectException(InvalidArgumentException::class);
+        $this->expectExceptionMessage('Incorrect number of arguments passed to filter');
+        $list->filter(...$args);
+    }
+
+    public function provideFilterByWrongNumArgs()
+    {
+        return [
+            0 => [],
+            3 => [1, 2, 3],
+        ];
+    }
+
+    /**
+     * @dataProvider provideLimitAndOffset
+     */
+    public function testLimitAndOffset($length, $offset, $expectedCount, $expectException = false)
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $this->assertSame(TeamComment::get()->count(), $list->count(), 'base count should match');
+
+        if ($expectException) {
+            $this->expectException(InvalidArgumentException::class);
+        }
+
+        $this->assertCount($expectedCount, $list->limit($length, $offset));
+        $this->assertCount(
+            $expectedCount,
+            $list->limit(0, 9999)->limit($length, $offset),
+            'Follow up limit calls unset previous ones'
+        );
+
+        // this mirrors an assertion in the tests for DataList to ensure they work the same way
+        $this->assertCount($expectedCount, $list->limit($length, $offset)->toArray());
+    }
+
+    public function provideLimitAndOffset(): array
+    {
+        return [
+            'no limit' => [null, 0, 3],
+            'smaller limit' => [2, 0, 2],
+            'greater limit' => [4, 0, 3],
+            'one limit' => [1, 0, 1],
+            'zero limit' => [0, 0, 0],
+            'limit and offset' => [1, 1, 1],
+            'false limit equivalent to 0' => [false, 0, 0],
+            'offset only' => [null, 2, 1],
+            'offset greater than list length' => [null, 3, 0],
+            'negative length' => [-1, 0, 0, true],
+            'negative offset' => [0, -1, 0, true],
+        ];
+    }
+
+    public function testToNestedArray()
+    {
+        $list = $this->getListWithRecords(TeamComment::class)->sort('ID');
+        $nestedArray = $list->toNestedArray();
+        $expected = [
+            [
+                'ClassName' => TeamComment::class,
+                'Name' => 'Joe',
+                'Comment' => 'This is a team comment by Joe',
+                'TeamID' => $this->objFromFixture(TeamComment::class, 'comment1')->TeamID,
+            ],
+            [
+                'ClassName' => TeamComment::class,
+                'Name' => 'Bob',
+                'Comment' => 'This is a team comment by Bob',
+                'TeamID' => $this->objFromFixture(TeamComment::class, 'comment2')->TeamID,
+            ],
+            [
+                'ClassName' => TeamComment::class,
+                'Name' => 'Phil',
+                'Comment' => 'Phil is a unique guy, and comments on team2',
+                'TeamID' => $this->objFromFixture(TeamComment::class, 'comment3')->TeamID,
+            ],
+        ];
+        $this->assertEquals(3, count($nestedArray ?? []));
+        $this->assertEquals($expected[0]['Name'], $nestedArray[0]['Name']);
+        $this->assertEquals($expected[1]['Comment'], $nestedArray[1]['Comment']);
+        $this->assertEquals($expected[2]['TeamID'], $nestedArray[2]['TeamID']);
+    }
+
+    public function testMap()
+    {
+        $map = $this->getListWithRecords(TeamComment::class)->map()->toArray();
+        $expected = [
+            $this->idFromFixture(TeamComment::class, 'comment1') => 'Joe',
+            $this->idFromFixture(TeamComment::class, 'comment2') => 'Bob',
+            $this->idFromFixture(TeamComment::class, 'comment3') => 'Phil'
+        ];
+
+        $this->assertEquals($expected, $map);
+        $otherMap = $this->getListWithRecords(TeamComment::class)->map('Name', 'TeamID')->toArray();
+        $otherExpected = [
+            'Joe' => $this->objFromFixture(TeamComment::class, 'comment1')->TeamID,
+            'Bob' => $this->objFromFixture(TeamComment::class, 'comment2')->TeamID,
+            'Phil' => $this->objFromFixture(TeamComment::class, 'comment3')->TeamID
+        ];
+
+        $this->assertEquals($otherExpected, $otherMap);
+    }
+
+    public function testAggregate()
+    {
+        // Test many_many_extraFields
+        $company = $this->objFromFixture(EquipmentCompany::class, 'equipmentcompany1');
+        $i = 0;
+        $sum = 0;
+        foreach ($company->SponsoredTeams() as $team) {
+            $i++;
+            $sum += $i;
+            $company->SponsoredTeams()->setExtraData($team->ID, ['SponsorFee' => $i]);
+        }
+
+        $teams = $this->getListWithRecords(
+            $company->SponsoredTeams(),
+            ManyManyList::class,
+            $company->ID,
+            [EquipmentCompany::class, 'SponsoredTeams']
+        );
+
+        // try with a field that is in $db
+        $this->assertEquals(7, $teams->max('NumericField'));
+        $this->assertEquals(2, $teams->min('NumericField'));
+        $this->assertEquals(4.5, $teams->avg('NumericField'));
+        $this->assertEquals(9, $teams->sum('NumericField'));
+        // try with a field from many_many_extraFields
+        $this->assertEquals($i, $teams->max('SponsorFee'));
+        $this->assertEquals(1, $teams->min('SponsorFee'));
+        $this->assertEquals(round($sum / $i, 4), round($teams->avg('SponsorFee'), 4));
+        $this->assertEquals($sum, $teams->sum('SponsorFee'));
+    }
+
+    public function testEach()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+
+        $count = 0;
+        $list->each(
+            function ($item) use (&$count) {
+                $count++;
+                $this->assertInstanceOf(TeamComment::class, $item);
+            }
+        );
+
+        $this->assertEquals($count, $list->count());
+    }
+
+    public function testByID()
+    {
+        // We can get a single item by ID.
+        $id = $this->idFromFixture(Team::class, 'team2');
+        $list = $this->getListWithRecords(Team::class);
+        $team = $list->byID($id);
+
+        // byID() returns a DataObject, rather than a list
+        $this->assertInstanceOf(Team::class, $team);
+        $this->assertEquals('Team 2', $team->Title);
+
+        // An invalid ID returns null
+        $this->assertNull($list->byID(0));
+        $this->assertNull($list->byID(-1));
+        $this->assertNull($list->byID(9999999));
+    }
+
+    public function testByIDs()
+    {
+        $knownIDs = $this->allFixtureIDs(Player::class);
+        $removedID = array_pop($knownIDs);
+        $expectedCount = count($knownIDs);
+        $list = $this->getListWithRecords(Player::class);
+
+        // Check we have all the players we searched for, and not the one we didn't
+        $filteredList = $list->byIDs($knownIDs);
+        foreach ($filteredList as $player) {
+            $this->assertContains($player->ID, $knownIDs);
+            $this->assertNotEquals($removedID, $player->ID);
+        }
+        $this->assertCount($expectedCount, $filteredList);
+
+        // Check we don't get an extra player when we include a non-existent ID in there
+        $knownIDs[] = 9999999;
+        $filteredList = $list->byIDs($knownIDs);
+        foreach ($filteredList as $player) {
+            $this->assertContains($player->ID, $knownIDs);
+            $this->assertNotEquals($removedID, $player->ID);
+            $this->assertNotEquals(9999999, $player->ID);
+        }
+        $this->assertCount($expectedCount, $filteredList);
+
+        // Check we don't include any records if searching against an empty list or non-existent ID
+        $this->assertEmpty($list->byIDs([]));
+        $this->assertEmpty($list->byIDs([9999999]));
+    }
+
+    public function testRemove()
+    {
+        $list = $this->getListWithRecords(Team::class);
+        $obj = $this->objFromFixture(Team::class, 'team2');
+
+        $this->assertTrue($list->hasID($obj->ID));
+        $list->remove($obj);
+        $this->assertFalse($list->hasID($obj->ID));
+    }
+
+    public function testCanSortBy()
+    {
+        // Basic check
+        $team = $this->getListWithRecords(Team::class);
+        $this->assertTrue($team->canSortBy('Title'));
+        $this->assertFalse($team->canSortBy('SubclassDatabaseField'));
+        $this->assertFalse($team->canSortBy('SomethingElse'));
+
+        // Subclasses
+        $subteam = $this->getListWithRecords(SubTeam::class);
+        $this->assertTrue($subteam->canSortBy('Title'));
+        $this->assertTrue($subteam->canSortBy('SubclassDatabaseField'));
+        $this->assertFalse($subteam->canSortBy('SomethingElse'));
+    }
+
+    public function testArrayAccess()
+    {
+        $list = $this->getListWithRecords(Team::class)->sort('Title');
+
+        // We can use array access to refer to single items in the EagerLoadedList, as if it were an array
+        $this->assertEquals('Subteam 1', $list[0]->Title);
+        $this->assertEquals('Subteam 3', $list[2]->Title);
+        $this->assertEquals('Team 2', $list[4]->Title);
+        $this->assertNull($list[9999]);
+    }
+
+    public function testFind()
+    {
+        $list = $this->getListWithRecords(Team::class);
+        $record = $list->find('Title', 'Team 1');
+        $this->assertEquals($this->idFromFixture(Team::class, 'team1'), $record->ID);
+        // Test that you get null for a non-match
+        $this->assertNull($list->find('Title', 'This team doesnt exist'));
+    }
+
+    public function testFindById()
+    {
+        $list = $this->getListWithRecords(Team::class);
+        $record = $list->find('ID', $this->idFromFixture(Team::class, 'team1'));
+        $this->assertEquals('Team 1', $record->Title);
+
+        // Test that you can call it twice on the same list
+        $record = $list->find('ID', $this->idFromFixture(Team::class, 'team2'));
+        $this->assertEquals('Team 2', $record->Title);
+
+        // Test that you get null for a non-match
+        $this->assertNull($list->find('ID', 9999999));
+    }
+
+    public function testSubtract()
+    {
+        $comment1 = $this->objFromFixture(TeamComment::class, 'comment1');
+        $subtractList = TeamComment::get()->filter('ID', $comment1->ID);
+        $fullList = TeamComment::get();
+        $newList = $fullList->subtract($subtractList);
+        $this->assertEquals(2, $newList->Count(), 'List should only contain two objects after subtraction');
+    }
+
+    public function testSubtractBadDataclassThrowsException()
+    {
+        $this->expectException(InvalidArgumentException::class);
+        $teamsComments = TeamComment::get();
+        $teams = Team::get();
+        $teamsComments->subtract($teams);
+    }
+
+    public function testSimpleSort()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->sort('Name');
+        $this->assertEquals('Bob', $list->first()->Name, 'First comment should be from Bob');
+        $this->assertEquals('Phil', $list->last()->Name, 'Last comment should be from Phil');
+    }
+
+    public function testSimpleSortOneArgumentASC()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->sort('Name ASC');
+        $this->assertEquals('Bob', $list->first()->Name, 'First comment should be from Bob');
+        $this->assertEquals('Phil', $list->last()->Name, 'Last comment should be from Phil');
+    }
+
+    public function testSimpleSortOneArgumentDESC()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->sort('Name DESC');
+        $this->assertEquals('Phil', $list->first()->Name, 'Last comment should be from Phil');
+        $this->assertEquals('Bob', $list->last()->Name, 'First comment should be from Bob');
+    }
+
+    public function testSortOneArgumentMultipleColumns()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->sort('TeamID ASC, Name DESC');
+        $this->assertEquals('Joe', $list->first()->Name, 'First comment should be from Bob');
+        $this->assertEquals('Phil', $list->last()->Name, 'Last comment should be from Phil');
+    }
+
+    public function testSimpleSortASC()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->sort('Name', 'asc');
+        $this->assertEquals('Bob', $list->first()->Name, 'First comment should be from Bob');
+        $this->assertEquals('Phil', $list->last()->Name, 'Last comment should be from Phil');
+    }
+
+    public function testSimpleSortDESC()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->sort('Name', 'desc');
+        $this->assertEquals('Phil', $list->first()->Name, 'Last comment should be from Phil');
+        $this->assertEquals('Bob', $list->last()->Name, 'First comment should be from Bob');
+    }
+
+    public function testSortWithArraySyntaxSortASC()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->sort(['Name'=>'asc']);
+        $this->assertEquals('Bob', $list->first()->Name, 'First comment should be from Bob');
+        $this->assertEquals('Phil', $list->last()->Name, 'Last comment should be from Phil');
+    }
+
+    public function testSortWithArraySyntaxSortDESC()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->sort(['Name'=>'desc']);
+        $this->assertEquals('Phil', $list->first()->Name, 'Last comment should be from Phil');
+        $this->assertEquals('Bob', $list->last()->Name, 'First comment should be from Bob');
+    }
+
+    public function testSortWithMultipleArraySyntaxSort()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->sort(['TeamID'=>'asc','Name'=>'desc']);
+        $this->assertEquals('Joe', $list->first()->Name, 'First comment should be from Bob');
+        $this->assertEquals('Phil', $list->last()->Name, 'Last comment should be from Phil');
+    }
+
+    public function testSortNumeric()
+    {
+        $list = $this->getListWithRecords(Sortable::class);
+        $list1 = $list->sort('Sort', 'ASC');
+        $this->assertEquals(
+            [
+            -10,
+            -2,
+            -1,
+            0,
+            1,
+            2,
+            10
+            ],
+            $list1->column('Sort')
+        );
+    }
+
+    public function testSortMixedCase()
+    {
+        $list = $this->getListWithRecords(Sortable::class);
+        $list1 = $list->sort('Name', 'ASC');
+        $this->assertEquals(
+            [
+            'Bob',
+            'bonny',
+            'jane',
+            'John',
+            'sam',
+            'Steve',
+            'steven'
+            ],
+            $list1->column('Name')
+        );
+    }
+
+    /**
+     * @dataProvider provideSortInvalidParameters
+     */
+    public function testSortInvalidParameters(string $sort, string $type): void
+    {
+        if ($type === 'valid') {
+            $this->expectNotToPerformAssertions();
+        } elseif ($type === 'invalid-direction') {
+            $this->expectException(InvalidArgumentException::class);
+            $this->expectExceptionMessageMatches('/Invalid sort direction/');
+        } elseif ($type === 'unknown-column') {
+            if (!(DB::get_conn()->getConnector() instanceof MySQLiConnector)) {
+                $this->markTestSkipped('Database connector is not MySQLiConnector');
+            }
+            $this->expectException(DatabaseException::class);
+            $this->expectExceptionMessageMatches('/Unknown column/');
+        } elseif ($type === 'invalid-column') {
+            $this->expectException(InvalidArgumentException::class);
+            $this->expectExceptionMessageMatches('/Invalid sort column/');
+        } elseif ($type === 'unknown-relation') {
+            $this->expectException(InvalidArgumentException::class);
+            $this->expectExceptionMessageMatches('/is not a relation on model/');
+        } elseif ($type === 'nonlinear-relation') {
+            $this->expectException(InvalidArgumentException::class);
+            $this->expectExceptionMessageMatches('/is not a linear relation on model/');
+        } else {
+            throw new \Exception("Invalid type $type");
+        }
+        // column('ID') is required because that triggers the actual sorting of the rows
+        $this->getListWithRecords(Team::class)->sort($sort)->column('ID');
+    }
+
+    /**
+     * @see DataListTest::provideRawSqlSortException()
+     */
+    public function provideSortInvalidParameters(): array
+    {
+        return [
+            ['Title', 'valid'],
+            ['Title asc', 'valid'],
+            ['"Title" ASC', 'valid'],
+            ['Title ASC, "DatabaseField"', 'valid'],
+            ['"Title", "DatabaseField" DESC', 'valid'],
+            ['Title ASC, DatabaseField DESC', 'valid'],
+            ['Title ASC, , DatabaseField DESC', 'invalid-column'],
+            ['"Captain"."ShirtNumber"', 'invalid-column'],
+            ['"Captain"."ShirtNumber" DESC', 'invalid-column'],
+            ['Title BACKWARDS', 'invalid-direction'],
+            ['"Strange non-existent column name"', 'invalid-column'],
+            ['NonExistentColumn', 'unknown-column'],
+            ['Team.NonExistentColumn', 'unknown-relation'],
+            ['"DataObjectTest_Team"."NonExistentColumn" ASC', 'invalid-column'],
+            ['"DataObjectTest_Team"."Title" ASC', 'invalid-column'],
+            ['DataObjectTest_Team.Title', 'unknown-relation'],
+            ['Title, 1 = 1', 'invalid-column'],
+            ["Title,'abc' = 'abc'", 'invalid-column'],
+            ['Title,Mod(ID,3)=1', 'invalid-column'],
+            ['(CASE WHEN ID < 3 THEN 1 ELSE 0 END)', 'invalid-column'],
+            ['Founder.Fans.Surname', 'nonlinear-relation'],
+        ];
+    }
+
+    /**
+     * @dataProvider provideSortDirectionValidationTwoArgs
+     */
+    public function testSortDirectionValidationTwoArgs(string $direction, string $type): void
+    {
+        if ($type === 'valid') {
+            $this->expectNotToPerformAssertions();
+        } else {
+            $this->expectException(InvalidArgumentException::class);
+            $this->expectExceptionMessageMatches('/Invalid sort direction/');
+        }
+        $this->getListWithRecords(Team::class)->sort('Title', $direction)->column('ID');
+    }
+
+    public function provideSortDirectionValidationTwoArgs(): array
+    {
+        return [
+            ['ASC', 'valid'],
+            ['asc', 'valid'],
+            ['DESC', 'valid'],
+            ['desc', 'valid'],
+            ['BACKWARDS', 'invalid'],
+        ];
+    }
+
+    /**
+     * Test passing scalar values to sort()
+     *
+     * @dataProvider provideSortScalarValues
+     */
+    public function testSortScalarValues(mixed $emtpyValue, string $type): void
+    {
+        $this->assertSame(['Subteam 1'], $this->getListWithRecords(Team::class)->limit(1)->column('Title'));
+        $list = $this->getListWithRecords(Team::class)->sort('Title DESC');
+        $this->assertSame(['Team 3'], $list->limit(1)->column('Title'));
+        $this->expectException(InvalidArgumentException::class);
+        if ($type === 'invalid-scalar') {
+            $this->expectExceptionMessage('sort() arguments must either be a string, an array, or null');
+        }
+        if ($type === 'empty-scalar') {
+            $this->expectExceptionMessage('Invalid sort parameter');
+        }
+
+        $list = $list->sort($emtpyValue);
+        $this->assertSame(['Subteam 1'], $list->limit(1)->column('Title'));
+    }
+
+    public function provideSortScalarValues(): array
+    {
+        return [
+            ['', 'empty-scalar'],
+            [[], 'empty-scalar'],
+            [false, 'invalid-scalar'],
+            [true, 'invalid-scalar'],
+            [0, 'invalid-scalar'],
+            [1, 'invalid-scalar'],
+        ];
+    }
+
+    /**
+     * Explicity tests that sort(null) will wipe any existing sort on a EagerLoadedList
+     */
+    public function testSortNull(): void
+    {
+        $order = Team::get()->column('ID');
+        $list = $this->getListWithRecords(Team::class)->sort('Title DESC');
+        $this->assertNotSame($order, $list->column('ID'));
+
+        $list = $list->sort(null);
+        $this->assertSame($order, $list->column('ID'));
+    }
+
+    public function provideSortMatchesDataList()
+    {
+        // These will be used to make fixtures
+        // We don't use a fixtures yaml file here because we want a full DataList of only
+        // records with THESE values, with no other items to interfere.
+        $dataSets = [
+            'numbers' => [
+                'field' => 'Sort',
+                'values' => [null, 0, 1, 123, 2, 3],
+            ],
+            'numeric-strings' => [
+                'field' => 'Name',
+                'values' => [null, '', '0', '1', '123', '2', '3'],
+            ],
+            'numeric-after-strings' => [
+                'field' => 'Name',
+                'values' => ['test1', 'test2', 'test0', 'test123', 'test3'],
+            ],
+            'strings' => [
+                'field' => 'Name',
+                'values' => [null, '', 'abc', 'a', 'A', 'AB', '1', '0'],
+            ],
+        ];
+
+        // Build the test scenario with both sort directions
+        $scenarios = [];
+        foreach (['ASC', 'DESC'] as $sortDir) {
+            foreach ($dataSets as $data) {
+                $scenarios[] = [
+                    'sortDir' => $sortDir,
+                    'field' => $data['field'],
+                    'values' => $data['values']
+                ];
+            }
+        }
+
+        return $scenarios;
+    }
+
+    /**
+     * @dataProvider provideSortMatchesDataList
+     */
+    public function testSortMatchesDataList(string $sortDir, string $field, array $values)
+    {
+        // Use explicit per-scenario fixtures
+        Sortable::get()->removeAll();
+        foreach ($values as $value) {
+            $data = [$field => $value];
+            if (!$field === 'Name') {
+                $data['Name'] = $value;
+            }
+            $record = new Sortable($data);
+            $record->write();
+        }
+
+        // Sort both a DataList and an EagerLoadedList by the same items
+        // and validate they have the same sort order
+        $dataList = Sortable::get()->sort([$field => $sortDir]);
+        $eagerList = $this->getListWithRecords(Sortable::class)->sort([$field => $sortDir]);
+        $this->assertSame($dataList->map('ID', $field)->toArray(), $eagerList->map('ID', $field)->toArray());
+    }
+
+    public function testCanFilterBy()
+    {
+        // Basic check
+        $team = $this->getListWithRecords(Team::class);
+        $this->assertTrue($team->canFilterBy("Title"));
+        $this->assertFalse($team->canFilterBy("SomethingElse"));
+
+        // Has one
+        $this->assertTrue($team->canFilterBy("CaptainID"));
+
+        // Subclasses
+        $subteam = $this->getListWithRecords(SubTeam::class);
+        $this->assertTrue($subteam->canFilterBy("Title"));
+        $this->assertTrue($subteam->canFilterBy("SubclassDatabaseField"));
+    }
+
+    public function testAddfilter()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->addFilter(['Name' => 'Bob']);
+        $this->assertEquals(1, $list->count());
+        $this->assertEquals('Bob', $list->first()->Name, 'Only comment should be from Bob');
+    }
+
+    public function testFilterAny()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->filterAny('Name', 'Bob');
+        $this->assertEquals(1, $list->count());
+    }
+
+    public function testFilterAnyMultipleArray()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->filterAny(['Name' => 'Bob', 'Comment' => 'This is a team comment by Bob']);
+        $this->assertEquals(1, $list->count());
+        $this->assertEquals('Bob', $list->first()->Name, 'Only comment should be from Bob');
+    }
+
+    public function testFilterAnyOnFilter()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->filter(
+            [
+                'TeamID' => $this->idFromFixture(Team::class, 'team1')
+            ]
+        );
+        $list = $list->filterAny(
+            [
+                'Name' => ['Phil', 'Joe'],
+                'Comment' => 'This is a team comment by Bob'
+            ]
+        );
+        $list = $list->sort('Name');
+        $this->assertEquals(2, $list->count());
+        $this->assertEquals(
+            'Bob',
+            $list->offsetGet(0)->Name,
+            'Results should include comments from Bob, matched by comment and team'
+        );
+        $this->assertEquals(
+            'Joe',
+            $list->offsetGet(1)->Name,
+            'Results should include comments by Joe, matched by name and team (not by comment)'
+        );
+
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->filter(
+            [
+                'TeamID' => $this->idFromFixture(Team::class, 'team1')
+            ]
+        );
+        $list = $list->filterAny(
+            [
+                'Name' => ['Phil', 'Joe'],
+                'Comment' => 'This is a team comment by Bob'
+            ]
+        );
+        $list = $list->sort('Name');
+        $list = $list->filter(['Name' => 'Bob']);
+        $this->assertEquals(1, $list->count());
+        $this->assertEquals(
+            'Bob',
+            $list->offsetGet(0)->Name,
+            'Results should include comments from Bob, matched by name and team'
+        );
+    }
+
+    public function testFilterAnyMultipleWithArrayFilter()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->filterAny(['Name' => ['Bob','Phil']]);
+        $this->assertEquals(2, $list->count(), 'There should be two comments');
+        $this->assertEquals('Bob', $list->first()->Name, 'First comment should be from Bob');
+        $this->assertEquals('Phil', $list->last()->Name, 'Last comment should be from Phil');
+    }
+
+    public function testFilterAnyArrayInArray()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->filterAny([
+            'Name' => ['Bob','Phil'],
+            'TeamID' => [$this->idFromFixture(Team::class, 'team1')]
+        ])->sort('Name');
+        $this->assertEquals(3, $list->count());
+        $this->assertEquals(
+            'Bob',
+            $list->offsetGet(0)->Name,
+            'Results should include comments from Bob, matched by name and team'
+        );
+        $this->assertEquals(
+            'Joe',
+            $list->offsetGet(1)->Name,
+            'Results should include comments by Joe, matched by team (not by name)'
+        );
+        $this->assertEquals(
+            'Phil',
+            $list->offsetGet(2)->Name,
+            'Results should include comments from Phil, matched by name (even if he\'s not in Team1)'
+        );
+    }
+
+    public function testFilterAndExcludeById()
+    {
+        $id = $this->idFromFixture(SubTeam::class, 'subteam1');
+        $list = $this->getListWithRecords(SubTeam::class)->filter('ID', $id);
+        $this->assertEquals($id, $list->first()->ID);
+
+        $list = $this->getListWithRecords(SubTeam::class);
+        $this->assertEquals(3, count($list ?? []));
+        $this->assertEquals(2, count($list->exclude('ID', $id) ?? []));
+    }
+
+    /**
+     * @dataProvider provideFilterByNull
+     */
+    public function testFilterByNull(string $filterMethod, array $filter, array $expected)
+    {
+        // Force DataObjectTest_Fan/fan5::Email to empty string
+        $fan5id = $this->idFromFixture(Fan::class, 'fan5');
+        DB::prepared_query("UPDATE \"DataObjectTest_Fan\" SET \"Email\" = '' WHERE \"ID\" = ?", [$fan5id]);
+        $list = $this->getListWithRecords(Fan::class);
+
+        $filteredList = $list->$filterMethod($filter);
+        $this->assertListEquals($expected, $filteredList);
+    }
+
+    public function provideFilterByNull()
+    {
+        return [
+            'Filter by null email' => [
+                'filterMethod' => 'filter',
+                'filter' => ['Email' => null],
+                'expected' => [
+                    [
+                        'Name' => 'Stephen',
+                    ],
+                    [
+                        'Name' => 'Mitch',
+                    ]
+                ],
+            ],
+            'Filter by empty only' => [
+                'filterMethod' => 'filter',
+                'filter' => ['Email' => ''],
+                'expected' => [
+                    [
+                        'Name' => 'Hamish',
+                    ]
+                ],
+            ],
+            'Filter by null or empty values' => [
+                'filterMethod' => 'filter',
+                'filter' => ['Email' => [null, '']],
+                'expected' => [
+                    [
+                        'Name' => 'Stephen',
+                    ],
+                    [
+                        'Name' => 'Mitch',
+                    ],
+                    [
+                        'Name' => 'Hamish',
+                    ]
+                ],
+            ],
+            'Filter by many including null, empty string, and non-empty' => [
+                'filterMethod' => 'filter',
+                'filter' => ['Email' => [null, '', 'damian@thefans.com']],
+                'expected' => [
+                    [
+                        'Name' => 'Damian',
+                        'Email' => 'damian@thefans.com',
+                    ],
+                    [
+                        'Name' => 'Stephen',
+                    ],
+                    [
+                        'Name' => 'Mitch',
+                    ],
+                    [
+                        'Name' => 'Hamish',
+                    ]
+                ],
+            ],
+            'Filter by many including empty string and non-empty' => [
+                'filterMethod' => 'filter',
+                'filter' => ['Email' => ['', 'damian@thefans.com']],
+                'expected' => [
+                    [
+                        'Name' => 'Damian',
+                        'Email' => 'damian@thefans.com',
+                    ],
+                    [
+                        'Name' => 'Hamish',
+                    ]
+                ],
+            ],
+        ];
+    }
+
+    public function testFilterByCallback()
+    {
+        $team1ID = $this->idFromFixture(Team::class, 'team1');
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->filterByCallback(
+            function ($item, $list) use ($team1ID) {
+                return $item->TeamID == $team1ID;
+            }
+        );
+
+        $result = $list->column('Name');
+        $expected = array_intersect($result ?? [], ['Joe', 'Bob']);
+
+        $this->assertEquals(2, $list->count());
+        $this->assertEquals($expected, $result, 'List should only contain comments from Team 1 (Joe and Bob)');
+        $this->assertTrue($list instanceof Filterable, 'The List should be of type SS_Filterable');
+    }
+
+    public function testSimpleExclude()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->exclude('Name', 'Bob');
+        $list = $list->sort('Name');
+        $this->assertEquals(2, $list->count());
+        $this->assertEquals('Joe', $list->first()->Name, 'First comment should be from Joe');
+        $this->assertEquals('Phil', $list->last()->Name, 'Last comment should be from Phil');
+    }
+
+    public function testSimpleExcludeWithMultiple()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->exclude('Name', ['Joe', 'Phil']);
+        $this->assertEquals(1, $list->count());
+        $this->assertEquals('Bob', $list->first()->Name, 'First comment should be from Bob');
+    }
+
+    public function testMultipleExcludeWithMiss()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->exclude(['Name' => 'Bob', 'Comment' => 'Does not match any comments']);
+        $this->assertEquals(3, $list->count());
+    }
+
+    public function testMultipleExclude()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->exclude(['Name' => 'Bob', 'Comment' => 'This is a team comment by Bob']);
+        $this->assertEquals(2, $list->count());
+    }
+
+    /**
+     * Test doesn't exclude if only matches one
+     */
+    public function testMultipleExcludeMultipleMatches()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->exclude(['Name' => 'Bob', 'Comment' => 'Phil is a unique guy, and comments on team2']);
+        $this->assertCount(3, $list);
+    }
+
+    /**
+     * exclude only those that match both
+     */
+    public function testMultipleExcludeArraysMultipleMatches()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->exclude([
+            'Name' => ['Bob', 'Phil'],
+            'Comment' => [
+                'This is a team comment by Bob',
+                'Phil is a unique guy, and comments on team2'
+            ]
+        ]);
+        $this->assertListEquals([['Name' => 'Joe']], $list);
+    }
+
+    /**
+     * Exclude only which matches both params
+     */
+    public function testMultipleExcludeArraysMultipleMatchesOneMiss()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->exclude([
+            'Name' => ['Bob', 'Phil'],
+            'Comment' => [
+                'Does not match any comments',
+                'Phil is a unique guy, and comments on team2'
+            ]
+        ]);
+        $list = $list->sort('Name');
+        $this->assertListEquals(
+            [
+                ['Name' => 'Bob'],
+                ['Name' => 'Joe'],
+            ],
+            $list
+        );
+    }
+
+    /**
+     * Test that if an exclude() is applied to a filter(), the filter() is still preserved.
+     * @dataProvider provideExcludeOnFilter
+     */
+    public function testExcludeOnFilter(array $filter, array $exclude, array $expected)
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->filter($filter);
+        $list = $list->exclude($exclude);
+        $this->assertListEquals($expected, $list->sort('Name'));
+    }
+
+    public function provideExcludeOnFilter()
+    {
+        return [
+            [
+                'filter' => ['Comment' => 'Phil is a unique guy, and comments on team2'],
+                'exclude' => ['Name' => 'Bob'],
+                'expected' => [
+                    ['Name' => 'Phil'],
+                ],
+            ],
+            [
+                'filter' => ['Name' => ['Phil', 'Bob']],
+                'exclude' => ['Name' => ['Bob', 'Joe']],
+                'expected' => [
+                    ['Name' => 'Phil'],
+                ],
+            ],
+            [
+                'filter' => ['Name' => ['Phil', 'Bob']],
+                'exclude' => [
+                    'Name' => ['Joe', 'Phil'],
+                    'Comment' => ['Matches no comments', 'Not a matching comment']
+                ],
+                'expected' => [
+                    ['Name' => 'Bob'],
+                    ['Name' => 'Phil'],
+                ],
+            ],
+        ];
+    }
+
+    /**
+     * Test that Bob and Phil are excluded (one match each)
+     */
+    public function testExcludeAny()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->excludeAny([
+            'Name' => 'Bob',
+            'Comment' => 'Phil is a unique guy, and comments on team2'
+        ]);
+        $this->assertListEquals([['Name' => 'Joe']], $list);
+    }
+
+    /**
+     * Test that Bob and Phil are excluded by Name
+     */
+    public function testExcludeAnyArrays()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->excludeAny([
+            'Name' => ['Bob', 'Phil'],
+            'Comment' => 'No matching comments'
+        ]);
+        $this->assertListEquals([['Name' => 'Joe']], $list);
+    }
+
+    /**
+     * Test that Bob is excluded by Name, Phil by comment
+     */
+    public function testExcludeAnyMultiArrays()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->excludeAny([
+            'Name' => ['Bob', 'Fred'],
+            'Comment' => ['No matching comments', 'Phil is a unique guy, and comments on team2']
+        ]);
+        $this->assertListEquals([['Name' => 'Joe']], $list);
+    }
+
+    public function testEmptyFilter()
+    {
+        $this->expectException(InvalidArgumentException::class);
+        $this->expectExceptionMessage('Cannot filter Name against an empty set');
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list->exclude('Name', []);
+    }
+
+    public function testMultipleExcludeWithMultipleThatCheersEitherTeam()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->exclude([
+            'Name' => 'Bob',
+            'TeamID' => [
+                $this->idFromFixture(Team::class, 'team1'),
+                $this->idFromFixture(Team::class, 'team2'),
+            ],
+        ]);
+        $list = $list->sort('Name');
+        $this->assertEquals(2, $list->count());
+        $this->assertEquals('Joe', $list->first()->Name, 'First comment should be from Phil');
+        $this->assertEquals('Phil', $list->last()->Name, 'First comment should be from Phil');
+    }
+
+    public function testMultipleExcludeWithMultipleThatCheersOnNonExistingTeam()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->exclude(['Name' => 'Bob', 'TeamID' => [3]]);
+        $this->assertEquals(3, $list->count());
+    }
+
+    public function testMultipleExcludeWithNoExclusion()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->exclude([
+            'Name' => ['Bob','Joe'],
+            'Comment' => 'Phil is a unique guy, and comments on team2',
+        ]);
+        $this->assertEquals(3, $list->count());
+    }
+
+    public function testMultipleExcludeWithTwoArray()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->exclude([
+            'Name' => ['Bob','Joe'],
+            'TeamID' => [
+                $this->idFromFixture(Team::class, 'team1'),
+                $this->idFromFixture(Team::class, 'team2'),
+            ],
+        ]);
+        $this->assertEquals(1, $list->count());
+        $this->assertEquals('Phil', $list->last()->Name, 'Only comment should be from Phil');
+    }
+
+    public function testMultipleExcludeWithTwoArrayOneTeam()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->exclude([
+            'Name' => ['Bob', 'Phil'],
+            'TeamID' => [$this->idFromFixture(Team::class, 'team1')],
+        ]);
+        $list = $list->sort('Name');
+        $this->assertEquals(2, $list->count());
+        $this->assertEquals('Joe', $list->first()->Name, 'First comment should be from Joe');
+        $this->assertEquals('Phil', $list->last()->Name, 'Last comment should be from Phil');
+    }
+
+    public function testReverse()
+    {
+        $list = $this->getListWithRecords(TeamComment::class);
+        $list = $list->sort('Name');
+        $list = $list->reverse();
+
+        $this->assertEquals('Bob', $list->last()->Name, 'Last comment should be from Bob');
+        $this->assertEquals('Phil', $list->first()->Name, 'First comment should be from Phil');
+    }
+
+    public function testShuffle()
+    {
+        // Try shuffling 3 times - it's technically possible the result of a shuffle could be
+        // the exact same order as the original list.
+        for ($attempts = 1; $attempts <= 3; $attempts++) {
+            $list = $this->getListWithRecords(Sortable::class)->shuffle();
+            $results1 = $list->column();
+            $results2 = $list->column();
+            // The lists should hold the same records
+            $this->assertSame(count($results1), count($results2));
+
+            $failed = false;
+            try {
+                // The list order should different each time we "execute" the list
+                $this->assertNotSame($results1, $results2);
+            } catch (ExpectationFailedException $e) {
+                $failed = true;
+                // Only fail the test if we've tried and failed 3 times.
+                if ($attempts === 3) {
+                    throw $e;
+                }
+            }
+
+            // If we've passed the shuffle test, don't retry.
+            if (!$failed) {
+                break;
+            }
+        }
+    }
+
+    public function testColumn()
+    {
+        // sorted so postgres won't complain about the order being different
+        $list = $this->getListWithRecords(RelationChildSecond::class)->sort('Title');
+        $ids = [
+            $this->idFromFixture(RelationChildSecond::class, 'test1'),
+            $this->idFromFixture(RelationChildSecond::class, 'test2'),
+            $this->idFromFixture(RelationChildSecond::class, 'test3'),
+            $this->idFromFixture(RelationChildSecond::class, 'test3-duplicate'),
+        ];
+
+        // Test default
+        $this->assertSame($ids, $list->column());
+
+        // Test specific field
+        $this->assertSame(['Test 1', 'Test 2', 'Test 3', 'Test 3'], $list->column('Title'));
+    }
+
+    public function testColumnUnique()
+    {
+        // sorted so postgres won't complain about the order being different
+        $list = $this->getListWithRecords(RelationChildSecond::class)->sort('Title');
+        $ids = [
+            $this->idFromFixture(RelationChildSecond::class, 'test1'),
+            $this->idFromFixture(RelationChildSecond::class, 'test2'),
+            $this->idFromFixture(RelationChildSecond::class, 'test3'),
+            $this->idFromFixture(RelationChildSecond::class, 'test3-duplicate'),
+        ];
+
+        // Test default
+        $this->assertSame($ids, $list->columnUnique());
+
+        // Test specific field
+        $this->assertSame(['Test 1', 'Test 2', 'Test 3'], $list->columnUnique('Title'));
+    }
+
+    public function testColumnFailureInvalidColumn()
+    {
+        $this->expectException(InvalidArgumentException::class);
+        $this->getListWithRecords(Category::class)->column('ObviouslyInvalidColumn');
+    }
+
+    public function testOffsetGet()
+    {
+        $list = $this->getListWithRecords(TeamComment::class)->sort('Name');
+        $this->assertEquals('Bob', $list->offsetGet(0)->Name);
+        $this->assertEquals('Joe', $list->offsetGet(1)->Name);
+        $this->assertEquals('Phil', $list->offsetGet(2)->Name);
+        $this->assertNull($list->offsetGet(999));
+    }
+
+    public function testOffsetExists()
+    {
+        $list = $this->getListWithRecords(TeamComment::class)->sort('Name');
+        $this->assertTrue($list->offsetExists(0));
+        $this->assertTrue($list->offsetExists(1));
+        $this->assertTrue($list->offsetExists(2));
+        $this->assertFalse($list->offsetExists(999));
+    }
+
+    public function testOffsetGetNegative()
+    {
+        $list = $this->getListWithRecords(TeamComment::class)->sort('Name');
+        $this->expectException(InvalidArgumentException::class);
+        $this->expectExceptionMessage('$offset can not be negative. -1 was provided.');
+        $list->offsetGet(-1);
+    }
+
+    public function testOffsetExistsNegative()
+    {
+        $list = $this->getListWithRecords(TeamComment::class)->sort('Name');
+        $this->expectException(InvalidArgumentException::class);
+        $this->expectExceptionMessage('$key can not be negative. -1 was provided.');
+        $list->offsetExists(-1);
+    }
+
+    public function testOffsetSet()
+    {
+        $list = $this->getListWithRecords(TeamComment::class)->sort('Name');
+        $this->expectException(BadMethodCallException::class);
+        $this->expectExceptionMessage("Can't alter items in an EagerLoadedList using array-access");
+        $list->offsetSet(0, null);
+    }
+
+    public function testOffsetUnset()
+    {
+        $list = $this->getListWithRecords(TeamComment::class)->sort('Name');
+        $this->expectException(BadMethodCallException::class);
+        $this->expectExceptionMessage("Can't alter items in an EagerLoadedList using array-access");
+        $list->offsetUnset(0);
+    }
+
+    /**
+     * @dataProvider provideRelation
+     */
+    public function testRelation(string $parentClass, string $relation, ?array $expected)
+    {
+        $relationList = $this->getListWithRecords($parentClass)->relation($relation);
+        if ($expected === null) {
+            $this->assertNull($relationList);
+        } else {
+            $this->assertInstanceOf(DataList::class, $relationList);
+            $this->assertListEquals($expected, $relationList);
+        }
+    }
+
+    /**
+     * @dataProvider provideRelation
+     */
+    public function testRelationEagerLoaded(string $parentClass, string $relation, ?array $expected, array $eagerLoaded)
+    {
+        // Get an EagerLoadedList and add the relation data to it
+        $list = $this->getListWithRecords($parentClass);
+        foreach ($eagerLoaded as $parentFixture => $childData) {
+            $parentID = $this->idFromFixture($parentClass, $parentFixture);
+            if ($expected === null) {
+                // has_one
+                $list->addEagerLoadedData($relation, $parentID, $this->objFromFixture($childData['class'], $childData['fixture']));
+            } else {
+                // has_many and many_many
+                $data = new EagerLoadedList($childData[0]['class'], DataList::class);
+                foreach ($childData as $child) {
+                    $childID = $this->idFromFixture($child['class'], $child['fixture']);
+                    $data->addRow(['ID' => $childID, 'Title' => $child['Title']]);
+                }
+                $list->addEagerLoadedData($relation, $parentID, $data);
+            }
+        }
+
+        // Test that eager loaded data is correctly fetched
+        $relationList = $list->relation($relation);
+        if ($expected === null) {
+            $this->assertNull($relationList);
+        } else {
+            $this->assertInstanceOf(EagerLoadedList::class, $relationList);
+            $this->assertListEquals($expected, $relationList);
+        }
+    }
+
+    public function provideRelation()
+    {
+        return [
+            'many_many' => [
+                'parentClass' => RelationChildFirst::class,
+                'relation' => 'ManyNext',
+                'expected' => [
+                    ['Title' => 'Test 1'],
+                    ['Title' => 'Test 2'],
+                    ['Title' => 'Test 3'],
+                ],
+                'eagerloaded' => [
+                    'test1' => [
+                        ['class' => RelationChildSecond::class, 'fixture' => 'test1', 'Title' => 'Test 1'],
+                        ['class' => RelationChildSecond::class, 'fixture' => 'test2', 'Title' => 'Test 2'],
+                    ],
+                    'test2' => [
+                        ['class' => RelationChildSecond::class, 'fixture' => 'test1', 'Title' => 'Test 1'],
+                        ['class' => RelationChildSecond::class, 'fixture' => 'test3', 'Title' => 'Test 3'],
+                    ],
+                ],
+            ],
+            'has_many' => [
+                'parentClass' => Team::class,
+                'relation' => 'SubTeams',
+                'expected' => [
+                    ['Title' => 'Subteam 1'],
+                ],
+                'eagerloaded' => [
+                    'team1' => [
+                        ['class' => SubTeam::class, 'fixture' => 'subteam1', 'Title' => 'Subteam 1'],
+                    ],
+                ],
+            ],
+            // calling relation() for a has_one just gives you null
+            'has_one' => [
+                'parentClass' => DataObjectTest\Company::class,
+                'relation' => 'Owner',
+                'expected' => null,
+                'eagerloaded' => [
+                    'company1' => [
+                        'class' => Player::class, 'fixture' => 'player1', 'Title' => 'Player 1',
+                    ],
+                    'company2' => [
+                        'class' => Player::class, 'fixture' => 'player2', 'Title' => 'Player 2',
+                    ],
+                ],
+            ],
+        ];
+    }
+
+    /**
+     * @dataProvider provideCreateDataObject
+     */
+    public function testCreateDataObject(string $dataClass, string $realClass, array $row)
+    {
+        $list = new EagerLoadedList($dataClass, DataList::class);
+
+        // ID key must be present
+        if (!array_key_exists('ID', $row)) {
+            $this->expectException(InvalidArgumentException::class);
+            $this->expectExceptionMessage('$row must have an ID');
+        }
+
+        $obj = $list->createDataObject($row);
+
+        // Validate the class is correct
+        $this->assertSame($realClass, get_class($obj));
+
+        // Validates all fields are available
+        foreach ($row as $field => $value) {
+            $this->assertSame($value, $obj->$field);
+        }
+    }
+
+    public function provideCreateDataObject()
+    {
+        return [
+            'no ClassName' => [
+                'dataClass' => Team::class,
+                'realClass' => Team::class,
+                'row' => [
+                    'ID' => 1,
+                    'Title' => 'Team 1',
+                    'NumericField' => '1',
+                    // Extra field that doesn't exist on that class
+                    'SubclassDatabaseField' => 'this shouldnt be there',
+                ],
+            ],
+            'subclassed ClassName' => [
+                'dataClass' => Team::class,
+                'realClass' => SubTeam::class,
+                'row' => [
+                    'ClassName' => SubTeam::class,
+                    'ID' => 1,
+                    'Title' => 'Team 1',
+                    'SubclassDatabaseField' => 'this time it should be there',
+                ],
+            ],
+            'RecordClassName takes precedence' => [
+                'dataClass' => Team::class,
+                'realClass' => SubTeam::class,
+                'row' => [
+                    'ClassName' => Player::class,
+                    'RecordClassName' => SubTeam::class,
+                    'ID' => 1,
+                    'Title' => 'Team 1',
+                    'SubclassDatabaseField' => 'this time it should be there',
+                ],
+            ],
+            'No ID' => [
+                'dataClass' => Team::class,
+                'realClass' => Team::class,
+                'row' => [
+                    'Title' => 'Team 1',
+                    'NumericField' => '1',
+                    'SubclassDatabaseField' => 'this shouldnt be there',
+                ],
+            ],
+        ];
+    }
+
+    public function testGetExtraFields()
+    {
+        // Prepare list
+        $manyManyComponent = DataObject::getSchema()->manyManyComponent(Team::class, 'Players');
+        $manyManyComponent['extraFields'] = DataObject::getSchema()->manyManyExtraFieldsForComponent(Team::class, 'Players');
+        $list = new EagerLoadedList(Player::class, ManyManyList::class, 9999, $manyManyComponent);
+
+        $team1 = $this->objFromFixture(Team::class, 'team1');
+        $expected = DataObject::getSchema()->manyManyExtraFieldsForComponent(Team::class, 'Players');
+        $this->assertSame($expected, $list->getExtraFields());
+    }
+
+    public function testGetExtraData()
+    {
+        // Prepare list
+        $manyManyComponent = DataObject::getSchema()->manyManyComponent(Team::class, 'Players');
+        $manyManyComponent['extraFields'] = DataObject::getSchema()->manyManyExtraFieldsForComponent(Team::class, 'Players');
+        $list = new EagerLoadedList(Player::class, ManyManyList::class, 9999, $manyManyComponent);
+
+        // Validate extra data
+        $row1 = [
+            'ID' => 1,
+            'Position' => 'Captain',
+        ];
+        $list->addRow($row1);
+        $this->assertEquals(['Position' => $row1['Position']], $list->getExtraData('Teams', $row1['ID']));
+        // Also check numeric string while we're at it
+        $this->assertEquals(['Position' => $row1['Position']], $list->getExtraData('Teams', (string)$row1['ID']));
+
+        // Validate no extra data
+        $row2 = [
+            'ID' => '2',
+        ];
+        $list->addRow($row2);
+        $this->assertEquals(['Position' => null], $list->getExtraData('Teams', $row2['ID']));
+
+        // Validate no record
+        $this->assertEquals([], $list->getExtraData('Teams', 99999));
+    }
+
+    public function testGetExtraDataBadID()
+    {
+        // Prepare list
+        $manyManyComponent = DataObject::getSchema()->manyManyComponent(Team::class, 'Players');
+        $manyManyComponent['extraFields'] = DataObject::getSchema()->manyManyExtraFieldsForComponent(Team::class, 'Players');
+        $list = new EagerLoadedList(Player::class, ManyManyList::class, 9999, $manyManyComponent);
+
+        // Test exception when ID not numeric
+        $this->expectException(InvalidArgumentException::class);
+        $this->expectExceptionMessage('$itemID must be an integer or numeric string');
+        $list->getExtraData('Teams', 'abc');
+    }
+
+    /**
+     * @dataProvider provideGetExtraDataBadListType
+     */
+    public function testGetExtraDataBadListType(string $listClass)
+    {
+        $list = new EagerLoadedList(Player::class, $listClass, 99999);
+        $this->expectException(BadMethodCallException::class);
+        $this->expectExceptionMessage('Cannot have extra fields on this list type');
+        $list->getExtraData('Teams', 1);
+    }
+
+    public function provideGetExtraDataBadListType()
+    {
+        return [
+            [HasManyList::class],
+            [DataList::class],
+        ];
+    }
+
+    public function testDebug()
+    {
+        $list = Sortable::get();
+
+        $result = $list->debug();
+        $this->assertStringStartsWith('<h2>' . DataList::class . '</h2>', $result);
+        $this->assertMatchesRegularExpression(
+            '/<ul>\s*(<li style="list-style-type: disc; margin-left: 20px">.*?<\/li>)+\s*<\/ul>/s',
+            $result
+        );
+        $this->assertStringEndsWith('</ul>', $result);
+    }
+}

tests/php/ORM/ManyManyListTest.php

@@ -2,11 +2,13 @@
 
 namespace SilverStripe\ORM\Tests;
 
+use InvalidArgumentException;
 use SilverStripe\Core\Config\Config;
 use SilverStripe\ORM\FieldType\DBMoney;
 use SilverStripe\ORM\ManyManyList;
 use SilverStripe\Core\Convert;
 use SilverStripe\Dev\SapphireTest;
+use SilverStripe\ORM\DataObject;
 use SilverStripe\ORM\Tests\DataObjectTest\Player;
 use SilverStripe\ORM\Tests\DataObjectTest\Team;
 use SilverStripe\ORM\Tests\ManyManyListTest\ExtraFieldsObject;
@@ -80,6 +82,42 @@ class ManyManyListTest extends SapphireTest
         $this->assertEquals('Test Product', $result->Title);
     }
 
+    public function testGetExtraFields()
+    {
+        $team1 = $this->objFromFixture(Team::class, 'team1');
+        $expected = DataObject::getSchema()->manyManyExtraFieldsForComponent(Team::class, 'Players');
+        $this->assertSame($expected, $team1->Players()->getExtraFields());
+    }
+
+    public function testGetExtraData()
+    {
+        // Get fixtures
+        $player1 = new Player();
+        $player1->write();
+        $player2 = new Player();
+        $player2->write();
+        $team1 = $this->objFromFixture(Team::class, 'team1');
+
+        // Validate extra data
+        $team1->Players()->add($player1, ['Position' => 'Captain']);
+        $this->assertEquals(['Position' => 'Captain'], $team1->Players()->getExtraData('Teams', $player1->ID));
+
+        // Validate no extra data
+        $team1->Players()->add($player2);
+        $this->assertEquals(['Position' => null], $team1->Players()->getExtraData('Teams', $player2->ID));
+
+        // Validate no record
+        $this->assertEquals([], $team1->Players()->getExtraData('Teams', 99999));
+    }
+
+    public function testGetExtraDataBadID()
+    {
+        $team1 = $this->objFromFixture(Team::class, 'team1');
+        $this->expectException(InvalidArgumentException::class);
+        $this->expectExceptionMessage('ManyManyList::getExtraData() passed a non-numeric child ID');
+        $team1->Players()->getExtraData('Teams', 'abc');
+    }
+
     public function testSetExtraData()
     {
         $obj = new ManyManyListTest\ExtraFieldsObject();