harukin/core
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix driver issue for large dbs

Browse Source 
add rudimentary resume
This commit is contained in:
Habeas Codice 2014-11-20 12:41:48 -08:00
parent 5ad79e3f04
commit 28a0209ffd
1 changed files with 79 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
install/migrate-mypg.php
View File

@ -57,8 +57,10 @@ function run_sql($file, $db, &$err, &$n) {
exit(); exit();
} }
foreach($sql as $stmt) { foreach($sql as $stmt) {
if($stmt == '' || $stmt == "\n" || $stmt == "\n\n") if($stmt == '' || $stmt == "\n" || $stmt == "\n\n") {
$c--;
continue; continue;
}
$r = $db->exec($stmt); $r = $db->exec($stmt);
if($r===false) { if($r===false) {
echo "\nError executing $stmt: ".var_export($db->errorInfo(), true)."\n"; echo "\nError executing $stmt: ".var_export($db->errorInfo(), true)."\n";
@ -109,7 +111,18 @@ foreach(array('install','include','mod','view') as $dir) {
} }
} }
$cfg = parse_htconfig('.htconfig.php'); $cfgfile = '.htconfig.php';
if($argv[1] == '--resume') {
if($argc < 4) {
echo "Resume usage {$argv[0]} --resume <table> <row>\n";
exit();
}
$starttable = $argv[2];
$startrow = $argv[3];
$cfgfile = '.htconfig.php-mysql';
}
$cfg = parse_htconfig($cfgfile);
$type = get_configtype($cfg); $type = get_configtype($cfg);
if($type != 'mysql') { if($type != 'mysql') {
echo "Error. Must start with standard mysql installation in .htconfig.php.\n"; echo "Error. Must start with standard mysql installation in .htconfig.php.\n";
@ -125,6 +138,10 @@ try {
exit(); exit();
} }
// mysql insists on buffering even when you use fetch() instead of fetchAll() for some stupid reason
// http://stackoverflow.com/questions/6895098/pdo-mysql-memory-consumption-with-large-result-set
$mydb->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false);
if(!file_exists('.htconfig.php-pgsql')) { if(!file_exists('.htconfig.php-pgsql')) {
echo "Enter postgres server info:\n"; echo "Enter postgres server info:\n";
$p['host'] = get_data("Hostname: ", '/[\w.]+/'); $p['host'] = get_data("Hostname: ", '/[\w.]+/');
@ -181,29 +198,38 @@ $H = "\033[0;35m";
$W = "\033[1;37m"; $W = "\033[1;37m";
$M = "\033[1;31m"; $M = "\033[1;31m";
$N = "\033[0m"; $N = "\033[0m";
$r = ask_question("Ready to migrate {$W}Red{$M}(#){$W}Matrix$N from mysql db @$B{$cfg['host']}$N/$B{$cfg['data']}$N to postgres db @$B{$pcfg['host']}$N/$B{$pcfg['data']}$N.
if(isset($starttable)) {
$r = ask_question("Ready to migrate {$W}Red{$M}(#){$W}Matrix$N from mysql db @$B{$cfg['host']}$N/$B{$cfg['data']}$N to postgres db @$B{$pcfg['host']}$N/$B{$pcfg['data']}$N.
Resuming failed migration ({$M}experimental$N) starting at table '$starttable' row $startrow.
Are you ready to begin (N/y)? ",
array('y', 'n'),
'n'
);
if($r == 'n')
exit();
} else {
$r = ask_question("Ready to migrate {$W}Red{$M}(#){$W}Matrix$N from mysql db @$B{$cfg['host']}$N/$B{$cfg['data']}$N to postgres db @$B{$pcfg['host']}$N/$B{$pcfg['data']}$N.
The site will be disabled during the migration by moving the $H.htconfig.php$N file to $H.htconfig.php-mysql$N. The site will be disabled during the migration by moving the $H.htconfig.php$N file to $H.htconfig.php-mysql$N.
If for any reason the migration fails, you will need to move the config file back into place manually before trying again. If for any reason the migration fails, you will need to move the config file back into place manually before trying again.
Are you ready to begin (N/y)? ", array('y','n'), 'n' Are you ready to begin (N/y)? ", array('y','n'), 'n'
);
if($r == 'n')
exit();
rename('.htconfig.php', '.htconfig.php-mysql');
run_sql('install/schema_postgres.sql', $pgdb, $err, $n);
if($err) {
echo "There were $err errors creating the pgsql schema. Unable to continue.\n";
/* $r = ask_question("There were $err errors (and $n ok) creating the schema. This shouldn't happen and continuing will probably fail.
Do you want to continue migration anyway (N/y)? ", array('y','n'), 'n'
); );
if($r == 'n')*/
exit();
}
echo "pgsql schema created. $n queries executed successfully.\n"; if($r == 'n')
exit();
rename('.htconfig.php', '.htconfig.php-mysql');
run_sql('install/schema_postgres.sql', $pgdb, $err, $n);
if($err) {
echo "There were $err errors creating the pgsql schema. Unable to continue.\n";
exit();
}
echo "pgsql schema created. $n queries executed successfully.\n";
}
$res = $pgdb->query("select relname, attname, pg_type.typname from ((pg_attribute inner join pg_class on attrelid=pg_class.oid) inner join pg_type on atttypid=pg_type.oid) inner join pg_namespace on relnamespace=pg_namespace.oid where nspname='public' and atttypid not in (26,27,28,29) and relkind='r' and attname <> 'item_search_vector';"); $res = $pgdb->query("select relname, attname, pg_type.typname from ((pg_attribute inner join pg_class on attrelid=pg_class.oid) inner join pg_type on atttypid=pg_type.oid) inner join pg_namespace on relnamespace=pg_namespace.oid where nspname='public' and atttypid not in (26,27,28,29) and relkind='r' and attname <> 'item_search_vector';");
if($res === false) { if($res === false) {
@ -215,9 +241,23 @@ $schema = array();
while(($row = $res->fetch()) !== false) while(($row = $res->fetch()) !== false)
$schema[$row[0]][$row[1]] = $row[2]; $schema[$row[0]][$row[1]] = $row[2];
$res = $pgdb->query("select relname, attname from pg_attribute inner join pg_class on attrelid=pg_class.oid inner join pg_constraint on conrelid=pg_class.oid and pg_attribute.attnum = any (conkey) where contype='p';");
if($res === false) {
echo "Error reading back primary keys. Unable to continue.\n";
var_export($pgdb->errorInfo());
exit();
}
$pkeys = array();
while(($row = $res->fetch()) !== false)
$pkeys[$row[0]] = $row[1];
$err = 0; $n = 0; $err = 0; $n = 0;
$reserved = array('ignore','key','with'); $reserved = array('ignore','key','with');
foreach($schema as $table=>$fields) { foreach($schema as $table=>$fields) {
if(isset($starttable) && !$n && $table != $starttable) {
echo "Skipping table $table\n";
continue;
}
$fnames = array_keys($fields); $fnames = array_keys($fields);
$pfnames = array_keys($fields); $pfnames = array_keys($fields);
@ -231,14 +271,16 @@ foreach($schema as $table=>$fields) {
$pfname = '"' . $pfname . '"'; $pfname = '"' . $pfname . '"';
$pfstr = implode(',', $pfnames); $pfstr = implode(',', $pfnames);
$res = $mydb->query("SELECT count(*) FROM $table;"); $cres = $mydb->query("SELECT count(*) FROM $table;");
if($res === false) { if($cres === false) {
echo "Fatal error counting table $table: ".var_export($mydb->errorInfo(), true)."\n"; echo "Fatal error counting table $table: ".var_export($mydb->errorInfo(), true)."\n";
exit(); exit();
} }
$nrows = $res->fetchColumn(0); $nrows = $cres->fetchColumn(0);
$cres->closeCursor();
if(!$nrows) { if(!$nrows) {
echo "\nTABLE $table has 0 rows in mysql db.\n"; echo "TABLE $table has 0 rows in mysql db.\n";
continue; continue;
} }
@ -246,10 +288,17 @@ foreach($schema as $table=>$fields) {
for($x=0, $c=count($fields); $x < $c; $x++) for($x=0, $c=count($fields); $x < $c; $x++)
$pstr .= ($x ? ',?' : '?'); $pstr .= ($x ? ',?' : '?');
$crow = 0; if(isset($starttable) && $table == $starttable) {
$selectsql = "SELECT $fstr FROM $table ORDER BY {$pkeys[$table]} LIMIT $nrows OFFSET $startrow;";
$crow = $startrow;
} else {
$selectsql = "SELECT $fstr FROM $table ORDER BY {$pkeys[$table]};";
$crow = 0;
}
echo "\033[255DTABLE: $table [$c fields] $crow/$nrows (".number_format(($crow/$nrows)*100,2)."%)\033[K"; echo "\033[255DTABLE: $table [$c fields] $crow/$nrows (".number_format(($crow/$nrows)*100,2)."%)\033[K";
$res = $mydb->query("SELECT $fstr FROM $table;"); $res = $mydb->query($selectsql);
if($res === false) { if($res === false) {
echo "Fatal Error importing table $table: ".var_export($mydb->errorInfo(), true)."\n"; echo "Fatal Error importing table $table: ".var_export($mydb->errorInfo(), true)."\n";
exit(); exit();
@ -273,14 +322,16 @@ foreach($schema as $table=>$fields) {
$r = $istmt->execute(); $r = $istmt->execute();
if($r === false) { if($r === false) {
$err++; $err++;
echo "\nInsert error: ".var_export(array($pgdb->errorInfo(), $table, $fields, $row), true)."\n"; echo "Insert error: ".var_export(array($pgdb->errorInfo(), $table, $fields, $row), true)."\nResume with {$argv[0]} --resume $table $crow\n";
//exit(); exit();
} else } else
$n++; $n++;
$crow++; $crow++;
if(($crow % 10) == 0 || $crow == $nrows) if(($crow % 10) == 0 || $crow == $nrows)
echo "\033[255DTABLE: $table [$c fields] $crow/$nrows (".number_format(($crow/$nrows)*100,2)."%)\033[K"; echo "\033[255DTABLE: $table [$c fields] $crow/$nrows (".number_format(($crow/$nrows)*100,2)."%)\033[K";
} }
$res->closeCursor();
echo "\n";
} }
echo "Done with $err errors and $n inserts.\n"; echo "Done with $err errors and $n inserts.\n";