diff --git a/README.md b/README.md index aa940af..ab0209e 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,20 @@ MySQLDump - PHP ========= -[Requirements](https://github.com/ifsnop/mysqldump-php#requirements) | -[Installing](https://github.com/ifsnop/mysqldump-php#installing) | -[Getting started](https://github.com/ifsnop/mysqldump-php#getting-started) | -[API](https://github.com/ifsnop/mysqldump-php#constructor-and-default-parameters) | -[Settings](https://github.com/ifsnop/mysqldump-php#dump-settings) | -[PDO Settings](https://github.com/ifsnop/mysqldump-php#pdo-settings) | -[TODO](https://github.com/ifsnop/mysqldump-php#todo) | -[License](https://github.com/ifsnop/mysqldump-php#license) | -[Credits](https://github.com/ifsnop/mysqldump-php#credits) +## Custom change on this fock: +* Add setting 'query_part_limit' for very big tables. It avoid the timeout problem when cannot select all records in a single query. +* Add print the dump to console, just leave the filename empty when call start + + +[Requirements](#requirements) | +[Installing](#installing) | +[Getting started](#getting-started) | +[API](#constructor-and-default-parameters) | +[Settings](#dump-settings) | +[PDO Settings](#pdo-settings) | +[TODO](#todo) | +[License](#license) | +[Credits](#credits) [![Build Status](https://travis-ci.org/ifsnop/mysqldump-php.svg?branch=devel)](https://travis-ci.org/ifsnop/mysqldump-php) [![Total Downloads](https://poser.pugx.org/ifsnop/mysqldump-php/downloads)](https://packagist.org/packages/ifsnop/mysqldump-php) @@ -217,6 +222,8 @@ $this->_dumpSettings = self::array_replace_recursive($dumpSettingsDefault, $dump ## Dump Settings +- **query_part_limit** + - Default is 5000. The script will select target table part by part each 5000 records until end of table instead of select all records in a single query. - **include-tables** - Only include these tables (array of table names), include all if empty. - **exclude-tables** diff --git a/src/Ifsnop/Mysqldump/Mysqldump.php b/src/Ifsnop/Mysqldump/Mysqldump.php index 106e1dc..35a6762 100644 --- a/src/Ifsnop/Mysqldump/Mysqldump.php +++ b/src/Ifsnop/Mysqldump/Mysqldump.php @@ -122,6 +122,7 @@ class Mysqldump 'compress' => Mysqldump::NONE, 'init_commands' => array(), 'no-data' => array(), + 'query_part_limit' => 5000, 'if-not-exists' => false, 'reset-auto-increment' => false, 'add-drop-database' => false, @@ -299,7 +300,7 @@ public function restore($path) $buffer = ''; while ( !feof($handle) ) { - $line = fgets($handle); + $line = trim(fgets($handle)); if (substr($line, 0, 2) == '--' || !$line) { continue; // skip comments @@ -379,12 +380,14 @@ protected function connect() case 'mysql': case 'pgsql': case 'dblib': + $this->dbHandler = @new PDO( $this->dsn, $this->user, $this->pass, $this->pdoSettings ); + // Execute init commands once connected foreach ($this->dumpSettings['init_commands'] as $stmt) { $this->dbHandler->exec($stmt); @@ -433,12 +436,6 @@ public function start($filename = '') // Write some basic info to output file $this->compressManager->write($this->getDumpFileHeader()); - // initiate a transaction at global level to create a consistent snapshot - if ($this->dumpSettings['single-transaction']) { - $this->dbHandler->exec($this->typeAdapter->setup_transaction()); - $this->dbHandler->exec($this->typeAdapter->start_transaction()); - } - // Store server settings and use sanner defaults to dump $this->compressManager->write( $this->typeAdapter->backup_parameters() @@ -490,12 +487,6 @@ public function start($filename = '') $this->compressManager->write( $this->typeAdapter->restore_parameters() ); - - // end transaction - if ($this->dumpSettings['single-transaction']) { - $this->dbHandler->exec($this->typeAdapter->commit_transaction()); - } - // Write some stats to output file. $this->compressManager->write($this->getDumpFileFooter()); // Close output file. @@ -706,8 +697,6 @@ private function matches($table, $arr) */ private function exportTables() { - - // Exporting tables one by one foreach ($this->tables as $table) { if ($this->matches($table, $this->dumpSettings['exclude-tables'])) { @@ -1133,6 +1122,53 @@ public function setInfoHook($callable) $this->infoCallable = $callable; } + private function listValuesParts($stmt, $limit, $offset, $tableName, &$onlyOnce, &$lineSize, &$colNames) + { + if ($limit !== false) { + $stmt .= " LIMIT {$limit} OFFSET $offset"; + } + + $resultSet = $this->dbHandler->query($stmt); + $resultSet->setFetchMode(PDO::FETCH_ASSOC); + + $ignore = $this->dumpSettings['insert-ignore'] ? ' IGNORE' : ''; + + $count = 0; + foreach ($resultSet as $row) { + $count++; + $vals = $this->prepareColumnValues($tableName, $row); + if ($onlyOnce || !$this->dumpSettings['extended-insert']) { + if ($this->dumpSettings['complete-insert']) { + $lineSize += $this->compressManager->write( + "INSERT$ignore INTO `$tableName` (". + implode(", ", $colNames). + ") VALUES (".implode(",", $vals).")" + ); + } else { + $lineSize += $this->compressManager->write( + "INSERT$ignore INTO `$tableName` VALUES (".implode(",", $vals).")" + ); + } + $onlyOnce = false; + } else { + $lineSize += $this->compressManager->write("," .PHP_EOL . "(".implode(",", $vals).")"); + } + if (($lineSize > $this->dumpSettings['net_buffer_length']) || + !$this->dumpSettings['extended-insert']) { + $onlyOnce = true; + $lineSize = $this->compressManager->write(";".PHP_EOL); + } + } + $resultSet->closeCursor(); + + if (!$onlyOnce) { + $this->compressManager->write(PHP_EOL."-- =================================".PHP_EOL); + } + + + return $count; + } + /** * Table rows extractor * @@ -1145,6 +1181,7 @@ private function listValues($tableName) $this->prepareListValues($tableName); $onlyOnce = true; + $lineSize = 0; // colStmt is used to form a query to obtain row values $colStmt = $this->getColumnStmt($tableName); @@ -1162,47 +1199,26 @@ private function listValues($tableName) $stmt .= " WHERE {$condition}"; } - $limit = $this->getTableLimit($tableName); + $tableLimit = $this->getTableLimit($tableName); - if ($limit !== false) { - $stmt .= " LIMIT {$limit}"; - } - - $resultSet = $this->dbHandler->query($stmt); - $resultSet->setFetchMode(PDO::FETCH_ASSOC); - - $ignore = $this->dumpSettings['insert-ignore'] ? ' IGNORE' : ''; + $limit = $this->dumpSettings['query_part_limit']; + $offset = 0; $count = 0; - $line = ''; - foreach ($resultSet as $row) { - $count++; - $vals = $this->prepareColumnValues($tableName, $row); - if ($onlyOnce || !$this->dumpSettings['extended-insert']) { - if ($this->dumpSettings['complete-insert']) { - $line .= "INSERT$ignore INTO `$tableName` (". - implode(", ", $colNames). - ") VALUES (".implode(",", $vals).")"; - } else { - $line .= "INSERT$ignore INTO `$tableName` VALUES (".implode(",", $vals).")"; - } - $onlyOnce = false; - } else { - $line .= ",(".implode(",", $vals).")"; - } - if ((strlen($line) > $this->dumpSettings['net_buffer_length']) || - !$this->dumpSettings['extended-insert']) { - $onlyOnce = true; - $this->compressManager->write($line . ";".PHP_EOL); - $line = ''; - } + + do + { + $aCount = $this->listValuesParts($stmt, $limit, $offset, $tableName, $onlyOnce, $lineSize, $colNames); + $count += $aCount; + $offset += $limit; } - $resultSet->closeCursor(); + while ($aCount > 0); - if ('' !== $line) { - $this->compressManager->write($line. ";".PHP_EOL); + if (!$onlyOnce) { + $this->compressManager->write(";".PHP_EOL); } + $this->endListValues($tableName, $count); if ($this->infoCallable) { @@ -1227,6 +1243,11 @@ public function prepareListValues($tableName) ); } + if ($this->dumpSettings['single-transaction']) { + $this->dbHandler->exec($this->typeAdapter->setup_transaction()); + $this->dbHandler->exec($this->typeAdapter->start_transaction()); + } + if ($this->dumpSettings['lock-tables'] && !$this->dumpSettings['single-transaction']) { $this->typeAdapter->lock_table($tableName); } @@ -1275,6 +1296,10 @@ public function endListValues($tableName, $count = 0) ); } + if ($this->dumpSettings['single-transaction']) { + $this->dbHandler->exec($this->typeAdapter->commit_transaction()); + } + if ($this->dumpSettings['lock-tables'] && !$this->dumpSettings['single-transaction']) { $this->typeAdapter->unlock_table($tableName); } @@ -1477,9 +1502,16 @@ class CompressNone extends CompressManagerFactory */ public function open($filename) { - $this->fileHandler = fopen($filename, "wb"); - if (false === $this->fileHandler) { - throw new Exception("Output file is not writable"); + if (empty($filename)) + { + $this->fileHandler = 0; + } + else + { + $this->fileHandler = fopen($filename, "wb"); + if (false === $this->fileHandler) { + throw new Exception("Output file is not writable"); + } } return true; @@ -1487,6 +1519,12 @@ public function open($filename) public function write($str) { + if (empty($this->fileHandler)) + { + echo $str; + return strlen($str); + } + $bytesWritten = fwrite($this->fileHandler, $str); if (false === $bytesWritten) { throw new Exception("Writting to file failed! Probably, there is no more free space left?"); @@ -1496,6 +1534,10 @@ public function write($str) public function close() { + if (empty($this->fileHandler)) + { + return true; + } return fclose($this->fileHandler); } }