From 7f2f6952fb96ebedc0cae86858c789a90ac60fdf Mon Sep 17 00:00:00 2001 From: Khary Sharpe Date: Tue, 5 Apr 2016 19:39:00 -0500 Subject: [PATCH 1/9] Allows developers to traverse the result via a 'cursor' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Efficiently process thousands of records, uses least amount of memory, hits the database once with single query, allows you to use a ‘cursor’ to process the result set via a callback/closure. --- src/Illuminate/Database/Connection.php | 26 ++++++++++++++++++++ src/Illuminate/Database/Eloquent/Builder.php | 26 ++++++++++++++++++++ src/Illuminate/Database/Query/Builder.php | 12 +++++++++ 3 files changed, 64 insertions(+) diff --git a/src/Illuminate/Database/Connection.php b/src/Illuminate/Database/Connection.php index f2a146f0ec95..22d6697dc27c 100755 --- a/src/Illuminate/Database/Connection.php +++ b/src/Illuminate/Database/Connection.php @@ -342,6 +342,32 @@ public function select($query, $bindings = [], $useReadPdo = true) }); } + /** + * Run a select statement against the database and return 'cursor' + * + * @param string $query + * @param array $bindings + * @param bool $useReadPdo + * @return mixed + */ + public function fetch($query, $bindings = [], $useReadPdo = true) + { + return $this->run($query, $bindings, function ($me, $query, $bindings) use ($useReadPdo) { + if ($me->pretending()) { + return []; + } + + // For select statements, we'll simply execute the query and return an array + // of the database result set. Each element in the array will be a single + // row from the database table, and will either be an array or objects. + $statement = $this->getPdoForSelect($useReadPdo)->prepare($query); + + $statement->execute($me->prepareBindings($bindings)); + + return $statement; + }); + } + /** * Get the PDO connection to use for a select query. * diff --git a/src/Illuminate/Database/Eloquent/Builder.php b/src/Illuminate/Database/Eloquent/Builder.php index 26ad9573f2e8..105e7458af94 100755 --- a/src/Illuminate/Database/Eloquent/Builder.php +++ b/src/Illuminate/Database/Eloquent/Builder.php @@ -290,6 +290,32 @@ public function firstOrFail($columns = ['*']) throw (new ModelNotFoundException)->setModel(get_class($this->model)); } + /** + * Fetch the row from the result set + * + * @param callable $callback + * @return bool + */ + public function fetch(callable $callback) + { + $builder = $this->applyScopes(); + + $statement = $this->query->fetch(); + + + while ($row = $statement->fetch()) { + // On each result set, we will pass them to the callback and then let the + // developer take care of everything within the callback, which allows us to + // keep the memory low for spinning through large result sets for working. + if (call_user_func($callback, $row) === false) { + return false; + } + + } + + return true; + } + /** * Execute the query as a "select" statement. * diff --git a/src/Illuminate/Database/Query/Builder.php b/src/Illuminate/Database/Query/Builder.php index 8e719b4c18b1..d7050d8978e9 100755 --- a/src/Illuminate/Database/Query/Builder.php +++ b/src/Illuminate/Database/Query/Builder.php @@ -1631,6 +1631,18 @@ protected function restoreFieldsForCount() $this->bindingBackups = []; } + /** + * Execute the query as a "select" statement. + * + * @return mixed + */ + public function fetch() + { + $results = $this->connection->fetch($this->toSql(), $this->getBindings(), ! $this->useWritePdo); + + return $results; + } + /** * Chunk the results of the query. * From f63eeadf58d0379a718ea6794f8ea3cf3ff06c8a Mon Sep 17 00:00:00 2001 From: Khary Sharpe Date: Tue, 5 Apr 2016 19:56:28 -0500 Subject: [PATCH 2/9] Fixes based on StyleCI feedback --- src/Illuminate/Database/Connection.php | 4 ++-- src/Illuminate/Database/Eloquent/Builder.php | 4 ++-- src/Illuminate/Database/Query/Builder.php | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Illuminate/Database/Connection.php b/src/Illuminate/Database/Connection.php index 22d6697dc27c..1ed34fbac2ff 100755 --- a/src/Illuminate/Database/Connection.php +++ b/src/Illuminate/Database/Connection.php @@ -343,7 +343,7 @@ public function select($query, $bindings = [], $useReadPdo = true) } /** - * Run a select statement against the database and return 'cursor' + * Run a select statement against the database and returns a 'cursor'. * * @param string $query * @param array $bindings @@ -366,7 +366,7 @@ public function fetch($query, $bindings = [], $useReadPdo = true) return $statement; }); - } + } /** * Get the PDO connection to use for a select query. diff --git a/src/Illuminate/Database/Eloquent/Builder.php b/src/Illuminate/Database/Eloquent/Builder.php index 105e7458af94..c25c9d41c9af 100755 --- a/src/Illuminate/Database/Eloquent/Builder.php +++ b/src/Illuminate/Database/Eloquent/Builder.php @@ -291,7 +291,7 @@ public function firstOrFail($columns = ['*']) } /** - * Fetch the row from the result set + * Fetch the row from the result set. * * @param callable $callback * @return bool @@ -314,7 +314,7 @@ public function fetch(callable $callback) } return true; - } + } /** * Execute the query as a "select" statement. diff --git a/src/Illuminate/Database/Query/Builder.php b/src/Illuminate/Database/Query/Builder.php index d7050d8978e9..4b4624fc9c4b 100755 --- a/src/Illuminate/Database/Query/Builder.php +++ b/src/Illuminate/Database/Query/Builder.php @@ -1638,7 +1638,7 @@ protected function restoreFieldsForCount() */ public function fetch() { - $results = $this->connection->fetch($this->toSql(), $this->getBindings(), ! $this->useWritePdo); + $results = $this->connection->fetch($this->toSql(), $this->getBindings(), ! $this->useWritePdo); return $results; } From ee9ec6348568ca5d58cc0e0b1df0f133da781c81 Mon Sep 17 00:00:00 2001 From: Khary Sharpe Date: Tue, 5 Apr 2016 19:59:07 -0500 Subject: [PATCH 3/9] Additional style fixes --- src/Illuminate/Database/Eloquent/Builder.php | 2 -- src/Illuminate/Database/Query/Builder.php | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Illuminate/Database/Eloquent/Builder.php b/src/Illuminate/Database/Eloquent/Builder.php index c25c9d41c9af..fb367baa8241 100755 --- a/src/Illuminate/Database/Eloquent/Builder.php +++ b/src/Illuminate/Database/Eloquent/Builder.php @@ -302,7 +302,6 @@ public function fetch(callable $callback) $statement = $this->query->fetch(); - while ($row = $statement->fetch()) { // On each result set, we will pass them to the callback and then let the // developer take care of everything within the callback, which allows us to @@ -310,7 +309,6 @@ public function fetch(callable $callback) if (call_user_func($callback, $row) === false) { return false; } - } return true; diff --git a/src/Illuminate/Database/Query/Builder.php b/src/Illuminate/Database/Query/Builder.php index 4b4624fc9c4b..b6a2180a30b6 100755 --- a/src/Illuminate/Database/Query/Builder.php +++ b/src/Illuminate/Database/Query/Builder.php @@ -1641,7 +1641,7 @@ public function fetch() $results = $this->connection->fetch($this->toSql(), $this->getBindings(), ! $this->useWritePdo); return $results; - } + } /** * Chunk the results of the query. From cc5faf612e5fc205a6be11ade8e8822c4e8c0951 Mon Sep 17 00:00:00 2001 From: Khary Sharpe Date: Wed, 6 Apr 2016 07:46:42 -0500 Subject: [PATCH 4/9] Name changed as suggested on Slack Set Fetch mode Switched to using generators --- src/Illuminate/Database/Connection.php | 2 ++ src/Illuminate/Database/Eloquent/Builder.php | 21 +++++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/Illuminate/Database/Connection.php b/src/Illuminate/Database/Connection.php index 1ed34fbac2ff..6d8a1019b969 100755 --- a/src/Illuminate/Database/Connection.php +++ b/src/Illuminate/Database/Connection.php @@ -362,6 +362,8 @@ public function fetch($query, $bindings = [], $useReadPdo = true) // row from the database table, and will either be an array or objects. $statement = $this->getPdoForSelect($useReadPdo)->prepare($query); + $statement->setFetchMode($me->getFetchMode()); + $statement->execute($me->prepareBindings($bindings)); return $statement; diff --git a/src/Illuminate/Database/Eloquent/Builder.php b/src/Illuminate/Database/Eloquent/Builder.php index fb367baa8241..c027d732ef2e 100755 --- a/src/Illuminate/Database/Eloquent/Builder.php +++ b/src/Illuminate/Database/Eloquent/Builder.php @@ -291,27 +291,30 @@ public function firstOrFail($columns = ['*']) } /** - * Fetch the row from the result set. - * - * @param callable $callback - * @return bool + * Traverses through a result set using a cursor. + * + * @return void */ - public function fetch(callable $callback) + public function traverse() { $builder = $this->applyScopes(); - $statement = $this->query->fetch(); + $statement = $builder->query->fetch(); + while ($row = $statement->fetch()) { // On each result set, we will pass them to the callback and then let the // developer take care of everything within the callback, which allows us to // keep the memory low for spinning through large result sets for working. - if (call_user_func($callback, $row) === false) { - return false; + + $continue = (yield $row); + + if ($continue === false) { + return; } + } - return true; } /** From dea6794dd321449dc98849627bdccf26074e181f Mon Sep 17 00:00:00 2001 From: Khary Sharpe Date: Wed, 6 Apr 2016 07:50:17 -0500 Subject: [PATCH 5/9] StyleCI fixes --- src/Illuminate/Database/Eloquent/Builder.php | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Illuminate/Database/Eloquent/Builder.php b/src/Illuminate/Database/Eloquent/Builder.php index c027d732ef2e..9cecef6f7969 100755 --- a/src/Illuminate/Database/Eloquent/Builder.php +++ b/src/Illuminate/Database/Eloquent/Builder.php @@ -292,8 +292,8 @@ public function firstOrFail($columns = ['*']) /** * Traverses through a result set using a cursor. - * - * @return void + * + * @return void */ public function traverse() { @@ -301,7 +301,6 @@ public function traverse() $statement = $builder->query->fetch(); - while ($row = $statement->fetch()) { // On each result set, we will pass them to the callback and then let the // developer take care of everything within the callback, which allows us to @@ -312,9 +311,7 @@ public function traverse() if ($continue === false) { return; } - } - } /** From dcdd22c7d5dfa29b75d526952d2f08e6c89eafaa Mon Sep 17 00:00:00 2001 From: Khary Sharpe Date: Wed, 6 Apr 2016 07:59:38 -0500 Subject: [PATCH 6/9] StyleCI grrr --- src/Illuminate/Database/Eloquent/Builder.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Illuminate/Database/Eloquent/Builder.php b/src/Illuminate/Database/Eloquent/Builder.php index 9cecef6f7969..1b21fb647bed 100755 --- a/src/Illuminate/Database/Eloquent/Builder.php +++ b/src/Illuminate/Database/Eloquent/Builder.php @@ -307,7 +307,7 @@ public function traverse() // keep the memory low for spinning through large result sets for working. $continue = (yield $row); - + if ($continue === false) { return; } From d1aafe0cb2efb9a0727b9bb4d0583db4012d29af Mon Sep 17 00:00:00 2001 From: Khary Sharpe Date: Tue, 5 Apr 2016 19:39:00 -0500 Subject: [PATCH 7/9] Allows developers to traverse the result set via a cursor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Efficiently process thousands of records, uses least amount of memory, hits the database once with single query, allows you to use a ‘cursor’ to traverse the result set. --- src/Illuminate/Database/Connection.php | 28 ++++++++++++++++++++ src/Illuminate/Database/Eloquent/Builder.php | 24 +++++++++++++++++ src/Illuminate/Database/Query/Builder.php | 12 +++++++++ 3 files changed, 64 insertions(+) diff --git a/src/Illuminate/Database/Connection.php b/src/Illuminate/Database/Connection.php index 6d8a1019b969..1cf9f235ffa2 100755 --- a/src/Illuminate/Database/Connection.php +++ b/src/Illuminate/Database/Connection.php @@ -370,6 +370,34 @@ public function fetch($query, $bindings = [], $useReadPdo = true) }); } + /** + * Run a select statement against the database and returns a 'cursor'. + * + * @param string $query + * @param array $bindings + * @param bool $useReadPdo + * @return mixed + */ + public function fetch($query, $bindings = [], $useReadPdo = true) + { + return $this->run($query, $bindings, function ($me, $query, $bindings) use ($useReadPdo) { + if ($me->pretending()) { + return []; + } + + // For select statements, we'll simply execute the query and return an array + // of the database result set. Each element in the array will be a single + // row from the database table, and will either be an array or objects. + $statement = $this->getPdoForSelect($useReadPdo)->prepare($query); + + $statement->setFetchMode($me->getFetchMode()); + + $statement->execute($me->prepareBindings($bindings)); + + return $statement; + }); + } + /** * Get the PDO connection to use for a select query. * diff --git a/src/Illuminate/Database/Eloquent/Builder.php b/src/Illuminate/Database/Eloquent/Builder.php index 1b21fb647bed..975669fafbb3 100755 --- a/src/Illuminate/Database/Eloquent/Builder.php +++ b/src/Illuminate/Database/Eloquent/Builder.php @@ -314,6 +314,30 @@ public function traverse() } } + /** + * Traverses through a result set using a cursor. + * + * @return void + */ + public function traverse() + { + $builder = $this->applyScopes(); + + $statement = $builder->query->fetch(); + + while ($row = $statement->fetch()) { + // On each result set, we will pass them to the callback and then let the + // developer take care of everything within the callback, which allows us to + // keep the memory low for spinning through large result sets for working. + + if ($row === false) { + return; + } + + yield $row; + } + } + /** * Execute the query as a "select" statement. * diff --git a/src/Illuminate/Database/Query/Builder.php b/src/Illuminate/Database/Query/Builder.php index b6a2180a30b6..fe2655b785a0 100755 --- a/src/Illuminate/Database/Query/Builder.php +++ b/src/Illuminate/Database/Query/Builder.php @@ -1643,6 +1643,18 @@ public function fetch() return $results; } + /** + * Execute the query as a "select" statement. + * + * @return mixed + */ + public function fetch() + { + $results = $this->connection->fetch($this->toSql(), $this->getBindings(), ! $this->useWritePdo); + + return $results; + } + /** * Chunk the results of the query. * From a2a1fb950a211c232904b81da7b2420b141d27ea Mon Sep 17 00:00:00 2001 From: Khary Sharpe Date: Wed, 6 Apr 2016 08:54:16 -0500 Subject: [PATCH 8/9] Fixed botched squash --- src/Illuminate/Database/Connection.php | 28 -------------------- src/Illuminate/Database/Eloquent/Builder.php | 24 ----------------- src/Illuminate/Database/Query/Builder.php | 12 --------- 3 files changed, 64 deletions(-) diff --git a/src/Illuminate/Database/Connection.php b/src/Illuminate/Database/Connection.php index 1cf9f235ffa2..6d8a1019b969 100755 --- a/src/Illuminate/Database/Connection.php +++ b/src/Illuminate/Database/Connection.php @@ -370,34 +370,6 @@ public function fetch($query, $bindings = [], $useReadPdo = true) }); } - /** - * Run a select statement against the database and returns a 'cursor'. - * - * @param string $query - * @param array $bindings - * @param bool $useReadPdo - * @return mixed - */ - public function fetch($query, $bindings = [], $useReadPdo = true) - { - return $this->run($query, $bindings, function ($me, $query, $bindings) use ($useReadPdo) { - if ($me->pretending()) { - return []; - } - - // For select statements, we'll simply execute the query and return an array - // of the database result set. Each element in the array will be a single - // row from the database table, and will either be an array or objects. - $statement = $this->getPdoForSelect($useReadPdo)->prepare($query); - - $statement->setFetchMode($me->getFetchMode()); - - $statement->execute($me->prepareBindings($bindings)); - - return $statement; - }); - } - /** * Get the PDO connection to use for a select query. * diff --git a/src/Illuminate/Database/Eloquent/Builder.php b/src/Illuminate/Database/Eloquent/Builder.php index 975669fafbb3..ddafb009feac 100755 --- a/src/Illuminate/Database/Eloquent/Builder.php +++ b/src/Illuminate/Database/Eloquent/Builder.php @@ -290,30 +290,6 @@ public function firstOrFail($columns = ['*']) throw (new ModelNotFoundException)->setModel(get_class($this->model)); } - /** - * Traverses through a result set using a cursor. - * - * @return void - */ - public function traverse() - { - $builder = $this->applyScopes(); - - $statement = $builder->query->fetch(); - - while ($row = $statement->fetch()) { - // On each result set, we will pass them to the callback and then let the - // developer take care of everything within the callback, which allows us to - // keep the memory low for spinning through large result sets for working. - - $continue = (yield $row); - - if ($continue === false) { - return; - } - } - } - /** * Traverses through a result set using a cursor. * diff --git a/src/Illuminate/Database/Query/Builder.php b/src/Illuminate/Database/Query/Builder.php index fe2655b785a0..b6a2180a30b6 100755 --- a/src/Illuminate/Database/Query/Builder.php +++ b/src/Illuminate/Database/Query/Builder.php @@ -1643,18 +1643,6 @@ public function fetch() return $results; } - /** - * Execute the query as a "select" statement. - * - * @return mixed - */ - public function fetch() - { - $results = $this->connection->fetch($this->toSql(), $this->getBindings(), ! $this->useWritePdo); - - return $results; - } - /** * Chunk the results of the query. * From cca72f729e9e35035005e93a57e85f1d8e2480d4 Mon Sep 17 00:00:00 2001 From: Khary Sharpe Date: Thu, 7 Apr 2016 07:16:43 -0500 Subject: [PATCH 9/9] Updated to return an eloquent model and changed naming to cursor. --- src/Illuminate/Database/Connection.php | 4 ++-- src/Illuminate/Database/Eloquent/Builder.php | 9 ++++++--- src/Illuminate/Database/Query/Builder.php | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Illuminate/Database/Connection.php b/src/Illuminate/Database/Connection.php index 6d8a1019b969..4cc72aee660f 100755 --- a/src/Illuminate/Database/Connection.php +++ b/src/Illuminate/Database/Connection.php @@ -343,14 +343,14 @@ public function select($query, $bindings = [], $useReadPdo = true) } /** - * Run a select statement against the database and returns a 'cursor'. + * Run a select statement against the database and returns a cursor. * * @param string $query * @param array $bindings * @param bool $useReadPdo * @return mixed */ - public function fetch($query, $bindings = [], $useReadPdo = true) + public function cursor($query, $bindings = [], $useReadPdo = true) { return $this->run($query, $bindings, function ($me, $query, $bindings) use ($useReadPdo) { if ($me->pretending()) { diff --git a/src/Illuminate/Database/Eloquent/Builder.php b/src/Illuminate/Database/Eloquent/Builder.php index ddafb009feac..d70a2cc1efdd 100755 --- a/src/Illuminate/Database/Eloquent/Builder.php +++ b/src/Illuminate/Database/Eloquent/Builder.php @@ -295,11 +295,11 @@ public function firstOrFail($columns = ['*']) * * @return void */ - public function traverse() + public function cursor() { $builder = $this->applyScopes(); - $statement = $builder->query->fetch(); + $statement = $builder->query->cursor(); while ($row = $statement->fetch()) { // On each result set, we will pass them to the callback and then let the @@ -310,7 +310,10 @@ public function traverse() return; } - yield $row; + //Hydrate and yield an Eloquent Model + $model = $this->model->newFromBuilder($row); + + yield $model; } } diff --git a/src/Illuminate/Database/Query/Builder.php b/src/Illuminate/Database/Query/Builder.php index b6a2180a30b6..a5ba58175343 100755 --- a/src/Illuminate/Database/Query/Builder.php +++ b/src/Illuminate/Database/Query/Builder.php @@ -1636,9 +1636,9 @@ protected function restoreFieldsForCount() * * @return mixed */ - public function fetch() + public function cursor() { - $results = $this->connection->fetch($this->toSql(), $this->getBindings(), ! $this->useWritePdo); + $results = $this->connection->cursor($this->toSql(), $this->getBindings(), ! $this->useWritePdo); return $results; }