omeka-s-modules / Scripto

Transcribe and translate items
GNU General Public License v3.0
7 stars 8 forks source link

Jobs may run out of memory for large instances #60

Closed jimsafley closed 5 years ago

jimsafley commented 5 years ago

The problem seems to come from ScriptoJob::getProjectItemIds() and ScriptoJob::getProjectMediaIds() when fetching the results using $query->getResult(). Since the queries are returning all rows, Doctrine's processing needs plus the resulting data structures can require enough memory to reach PHP's limit. We need to find a way to reduce memory usage. Things to try:

diff --git a/src/Job/ScriptoJob.php b/src/Job/ScriptoJob.php
index 7b2093c..b2643df 100644
--- a/src/Job/ScriptoJob.php
+++ b/src/Job/ScriptoJob.php
@@ -22,8 +22,16 @@ abstract class ScriptoJob extends AbstractJob
             JOIN si.item i
             JOIN si.scriptoProject sp
             WHERE sp.id = :scripto_project_id'
-        )->setParameter('scripto_project_id', $project->getId());
-        return array_column($query->getResult(), 'item_id', 'scripto_item_id');
+        );
+        $conn = $em->getConnection();
+        $stmt = $conn->prepare($query->getSQL());
+        $stmt->bindValue(1, $project->getId());
+        $stmt->execute();
+        $results = [];
+        foreach ($stmt as $row) {
+            $results[$row['id_0']] = $row['id_1'];
+        }
+        return $results;
     }

     /**
@@ -42,8 +50,16 @@ abstract class ScriptoJob extends AbstractJob
             JOIN sm.scriptoItem si
             JOIN si.scriptoProject sp
             WHERE sp.id = :scripto_project_id'
-        )->setParameter('scripto_project_id', $project->getId());
-        return array_column($query->getResult(), 'media_id', 'scripto_media_id');
+        );
+        $conn = $em->getConnection();
+        $stmt = $conn->prepare($query->getSQL());
+        $stmt->bindValue(1, $project->getId());
+        $stmt->execute();
+        $results = [];
+        foreach ($stmt as $row) {
+            $results[$row['id_0']] = $row['id_1'];
+        }
+        return $results;
     }

     /**