tonyair/silverstripe-framework
1
0
Fork 0
mirror of https://github.com/silverstripe/silverstripe-framework synced 2024-10-22 14:05:37 +02:00
Code Issues Projects Releases Wiki Activity

FIX Resolve problems with eagerloading performance

Browse Source
This commit is contained in:
Guy Sartorelli 2023-07-03 18:07:28 +12:00
parent 85e503d012
commit 7af0fe245c
No known key found for this signature in database
GPG Key ID: F313E3B9504D496A
1 changed files with 97 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

187
src/ORM/DataList.php
View File

@ -1072,13 +1072,10 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
if (empty($this->eagerLoadRelations)) { if (empty($this->eagerLoadRelations)) {
return; return;
} }
$ids = $query->column('ID'); $topLevelIDs = $query->column('ID');
if (empty($ids)) { if (empty($topLevelIDs)) {
return; return;
} }
$topLevelIDs = $ids;
// Using ->toArray() and then iterating instead of just iterating DataList because
// in some instances it prevents some extra SQL queries
$prevRelationArray = []; $prevRelationArray = [];
foreach ($this->eagerLoadRelations as $eagerLoadRelation) { foreach ($this->eagerLoadRelations as $eagerLoadRelation) {
list( list(
@ -1089,52 +1086,51 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
$hasManyIDField, $hasManyIDField,
$manyManyLastComponent $manyManyLastComponent
) = $this->getEagerLoadVariables($eagerLoadRelation); ) = $this->getEagerLoadVariables($eagerLoadRelation);
$dataClass = $dataClasses[count($dataClasses) - 2]; $parentDataClass = $dataClasses[count($dataClasses) - 2];
$relation = $relations[count($relations) - 1]; $relationName = $relations[count($relations) - 1];
$relationDataClass = $dataClasses[count($dataClasses) - 1]; $relationDataClass = $dataClasses[count($dataClasses) - 1];
if ($dataClass === $this->dataClass) { if ($parentDataClass === $this->dataClass()) {
// When we're at "the top of a tree of nested relationships", we can just use the IDs from the query // When we're at "the top of a tree of nested relationships", we can just use the IDs from the query
// This is important to do when handling multiple eager-loaded relatioship trees. // This is important to do when handling multiple eager-loaded relationship trees.
$ids = $topLevelIDs; $parentIDs = $topLevelIDs;
} }
// has_one // has_one
if ($hasOneIDField) { if ($hasOneIDField) {
list($prevRelationArray, $ids) = $this->fetchEagerLoadHasOne( list($prevRelationArray, $parentIDs) = $this->fetchEagerLoadHasOne(
$query, $query,
$prevRelationArray, $prevRelationArray,
$hasOneIDField, $hasOneIDField,
$relationDataClass, $relationDataClass,
$eagerLoadRelation, $eagerLoadRelation,
$relation, $relationName,
$dataClass, $parentDataClass
$dataClasses
); );
// belongs_to // belongs_to
} elseif ($belongsToIDField) { } elseif ($belongsToIDField) {
list($prevRelationArray, $ids) = $this->fetchEagerLoadBelongsTo( list($prevRelationArray, $parentIDs) = $this->fetchEagerLoadBelongsTo(
$ids, $parentIDs,
$belongsToIDField, $belongsToIDField,
$relationDataClass, $relationDataClass,
$eagerLoadRelation, $eagerLoadRelation,
$relation $relationName
); );
// has_many // has_many
} elseif ($hasManyIDField) { } elseif ($hasManyIDField) {
list($prevRelationArray, $ids) = $this->fetchEagerLoadHasMany( list($prevRelationArray, $parentIDs) = $this->fetchEagerLoadHasMany(
$ids, $parentIDs,
$hasManyIDField, $hasManyIDField,
$relationDataClass, $relationDataClass,
$eagerLoadRelation, $eagerLoadRelation,
$relation $relationName
); );
// many_many + belongs_many_many & many_many_through // many_many + belongs_many_many & many_many_through
} elseif ($manyManyLastComponent) { } elseif ($manyManyLastComponent) {
list($prevRelationArray, $ids) = $this->fetchEagerLoadManyMany( list($prevRelationArray, $parentIDs) = $this->fetchEagerLoadManyMany(
$manyManyLastComponent, $manyManyLastComponent,
$ids, $parentIDs,
$relationDataClass, $relationDataClass,
$eagerLoadRelation, $eagerLoadRelation,
$relation $relationName
); );
} else { } else {
throw new LogicException('Something went wrong with the eager loading'); throw new LogicException('Something went wrong with the eager loading');
@ -1144,27 +1140,28 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
private function fetchEagerLoadHasOne( private function fetchEagerLoadHasOne(
Query $query, Query $query,
array $prevRelationArray, array $parentRecords,
string $hasOneIDField, string $hasOneIDField,
string $relationDataClass, string $relationDataClass,
string $eagerLoadRelation, string $eagerLoadRelation,
string $relation, string $relationName,
string $dataClass, string $parentDataClass
array $dataClasses ): array {
): array
{
$itemArray = []; $itemArray = [];
$relationItemIDs = []; $relationItemIDs = [];
if ($dataClass === $dataClasses[0]) {
while ($row = $query->record()) { // It's a has_one directly on the records in THIS list
if ($parentDataClass === $this->dataClass()) {
foreach ($query as $itemData) {
$itemArray[] = [ $itemArray[] = [
'ID' => $row['ID'], 'ID' => $itemData['ID'],
$hasOneIDField => $row[$hasOneIDField] $hasOneIDField => $itemData[$hasOneIDField]
]; ];
$relationItemIDs[] = $row[$hasOneIDField]; $relationItemIDs[] = $itemData[$hasOneIDField];
} }
// It's a has_one on a list we've already eager-loaded
} else { } else {
foreach ($prevRelationArray as $itemData) { foreach ($parentRecords as $itemData) {
$itemArray[] = [ $itemArray[] = [
'ID' => $itemData->ID, 'ID' => $itemData->ID,
$hasOneIDField => $itemData->$hasOneIDField $hasOneIDField => $itemData->$hasOneIDField
@ -1172,12 +1169,12 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
$relationItemIDs[] = $itemData->$hasOneIDField; $relationItemIDs[] = $itemData->$hasOneIDField;
} }
} }
$relationArray = DataObject::get($relationDataClass)->filter(['ID' => $relationItemIDs])->toArray(); $relationArray = DataObject::get($relationDataClass)->byIDs($relationItemIDs)->toArray();
foreach ($itemArray as $itemData) { foreach ($itemArray as $itemData) {
foreach ($relationArray as $relationItem) { foreach ($relationArray as $relationItem) {
$eagerLoadID = $itemData['ID']; $eagerLoadID = $itemData['ID'];
if ($relationItem->ID === $itemData[$hasOneIDField]) { if ($relationItem->ID === $itemData[$hasOneIDField]) {
$this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation] = $relationItem; $this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relationName] = $relationItem;
} }
} }
} }
@ -1185,98 +1182,107 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
} }
private function fetchEagerLoadBelongsTo( private function fetchEagerLoadBelongsTo(
array $ids, array $parentIDs,
string $belongsToIDField, string $belongsToIDField,
string $relationDataClass, string $relationDataClass,
string $eagerLoadRelation, string $eagerLoadRelation,
string $relation string $relationName
): array ): array {
{ // Get ALL of the items for this relation up front, for ALL of the parents
$relationArray = DataObject::get($relationDataClass)->filter([$belongsToIDField => $ids])->toArray(); // Fetched as an array to avoid sporadic additional queries when the DataList is looped directly
$relationArray = DataObject::get($relationDataClass)->filter([$belongsToIDField => $parentIDs])->toArray();
$relationItemIDs = []; $relationItemIDs = [];
// Store the children against the correct parent
foreach ($relationArray as $relationItem) { foreach ($relationArray as $relationItem) {
$relationItemIDs[] = $relationItem->ID; $relationItemIDs[] = $relationItem->ID;
$eagerLoadID = $relationItem->$belongsToIDField; $eagerLoadID = $relationItem->$belongsToIDField;
$this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation] = $relationItem; $this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relationName] = $relationItem;
} }
return [$relationArray, $relationItemIDs]; return [$relationArray, $relationItemIDs];
} }
private function fetchEagerLoadHasMany( private function fetchEagerLoadHasMany(
array $ids, array $parentIDs,
string $hasManyIDField, string $hasManyIDField,
string $relationDataClass, string $relationDataClass,
string $eagerLoadRelation, string $eagerLoadRelation,
string $relation string $relationName
): array ): array {
{ // Get ALL of the items for this relation up front, for ALL of the parents
$relationArray = DataObject::get($relationDataClass)->filter([$hasManyIDField => $ids])->toArray(); // Fetched as an array to avoid sporadic additional queries when the DataList is looped directly
$relationArray = DataObject::get($relationDataClass)->filter([$hasManyIDField => $parentIDs])->toArray();
$relationItemIDs = []; $relationItemIDs = [];
// Store the children in an ArrayList against the correct parent
foreach ($relationArray as $relationItem) { foreach ($relationArray as $relationItem) {
$relationItemIDs[] = $relationItem->ID; $relationItemIDs[] = $relationItem->ID;
$eagerLoadID = $relationItem->$hasManyIDField; $eagerLoadID = $relationItem->$hasManyIDField;
if (!isset($this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation])) { if (!isset($this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relationName])) {
$arrayList = ArrayList::create(); $arrayList = ArrayList::create();
$arrayList->setDataClass($relationItem->dataClass); $arrayList->setDataClass($relationItem->dataClass);
$this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation] = $arrayList; $this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relationName] = $arrayList;
} }
$this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation]->push($relationItem); $this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relationName]->push($relationItem);
} }
return [$relationArray, $relationItemIDs]; return [$relationArray, $relationItemIDs];
} }
private function fetchEagerLoadManyMany( private function fetchEagerLoadManyMany(
array $manyManyLastComponent, array $manyManyLastComponent,
array $ids, array $parentIDs,
string $relationDataClass, string $relationDataClass,
string $eagerLoadRelation, string $eagerLoadRelation,
string $relation string $relationName
): array ): array {
{ $parentIDField = $manyManyLastComponent['parentField'];
$parentField = $manyManyLastComponent['parentField']; $childIDField = $manyManyLastComponent['childField'];
$childField = $manyManyLastComponent['childField'];
// $join will either be: // $join will either be:
// - the join table name for many-many // - the join table name for many-many
// - the join data class for many-many-through // - the join data class for many-many-through
$join = $manyManyLastComponent['join']; $join = $manyManyLastComponent['join'];
// many_many_through // many_many_through
if (is_a($manyManyLastComponent['relationClass'], ManyManyThroughList::class, true)) { if (is_a($manyManyLastComponent['relationClass'], ManyManyThroughList::class, true)) {
$joinThroughObjs = $join::get()->filter([$parentField => $ids]); $joinThroughObjs = DataObject::get($join)->filter([$parentIDField => $parentIDs]);
$relationItemIDs = []; $relationItemIDs = [];
$rows = []; $joinRows = [];
foreach ($joinThroughObjs as $joinThroughObj) { foreach ($joinThroughObjs as $joinThroughObj) {
$rows[] = [ $joinRows[] = [
$parentField => $joinThroughObj->$parentField, $parentIDField => $joinThroughObj->$parentIDField,
$childField => $joinThroughObj->$childField $childIDField => $joinThroughObj->$childIDField
]; ];
$relationItemIDs[] = $joinThroughObj->$childField; $relationItemIDs[] = $joinThroughObj->$childIDField;
} }
// many_many + belongs_many_many // many_many + belongs_many_many
} else { } else {
$joinTableQuery = DB::query('SELECT * FROM "' . $join . '" WHERE "' . $parentField . '" IN (' . implode(',', $ids) . ')'); $joinTableQuery = DB::query('SELECT * FROM "' . $join . '" WHERE "' . $parentIDField . '" IN (' . implode(',', $parentIDs) . ')');
$relationItemIDs = []; $relationItemIDs = $joinTableQuery->column($childIDField);
$rows = []; $joinRows = $joinTableQuery;
while ($row = $joinTableQuery->record()) {
$rows[] = [
$parentField => $row[$parentField],
$childField => $row[$childField]
];
$relationItemIDs[] = $row[$childField];
} }
}
$relationArray = DataObject::get($relationDataClass)->filter(['ID' => $relationItemIDs])->toArray(); // Get ALL of the items for this relation up front, for ALL of the parents
foreach ($rows as $row) { // Fetched as a map so we can get the ID for all records up front (instead of in another nested loop)
$eagerLoadID = $row[$parentField]; // Fetched after that as an array because for some reason that performs better in the loop
if (!isset($this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation])) { // Note that "Me" is a method on ViewableData that returns $this - i.e. that is the actual DataObject record
$relationArray = DataObject::get($relationDataClass)->byIDs($relationItemIDs)->map('ID', 'Me')->toArray();
// Store the children in an ArrayList against the correct parent
foreach ($joinRows as $row) {
$parentID = $row[$parentIDField];
$childID = $row[$childIDField];
$relationItem = $relationArray[$childID];
if (!isset($this->eagerLoadedData[$eagerLoadRelation][$parentID][$relationName])) {
$arrayList = ArrayList::create(); $arrayList = ArrayList::create();
$arrayList->setDataClass($manyManyLastComponent['childClass']); $arrayList->setDataClass($relationItem->dataClass);
$this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation] = $arrayList; $this->eagerLoadedData[$eagerLoadRelation][$parentID][$relationName] = $arrayList;
} }
$relationItem = array_values(array_filter($relationArray, function ($relationItem) use ($row, $childField) { $this->eagerLoadedData[$eagerLoadRelation][$parentID][$relationName]->push($relationItem);
return $relationItem->ID === $row[$childField];
}))[0];
$this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation]->push($relationItem);
} }
return [$relationArray, $relationItemIDs]; return [$relationArray, $relationItemIDs];
} }
@ -1305,15 +1311,16 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
$message = "Eager loading only supports up to 3 levels of nesting, passed $count levels - $relation"; $message = "Eager loading only supports up to 3 levels of nesting, passed $count levels - $relation";
throw new InvalidArgumentException($message); throw new InvalidArgumentException($message);
} }
for ($i = 0; $i < $count; $i++) { // Add each relation in the chain as its own entry to be eagerloaded
if ($i === 0) { // e.g. for "Players.Teams.Coaches" you'll have three entries:
$arr[] = $parts[$i]; // "Players", "Players.Teams", and "Players.Teams.Coaches
} else { $usedParts = [];
$arr[] = $arr[count($arr) - 1] . '.' . $parts[$i]; foreach ($parts as $part) {
$usedParts[] = $part;
$arr[] = implode('.', $usedParts);
} }
} }
} $this->eagerLoadRelations = array_unique(array_merge($this->eagerLoadRelations, $arr));
$this->eagerLoadRelations = array_merge($this->eagerLoadRelations, $arr);
return $this; return $this;
} }