Refactored the scan and import functions to reduce redundant code and simplify logic. Import of pages, layouts, and blocks works.

This commit is contained in:
Andrew Manning 2016-07-14 22:24:23 -04:00
parent 960e9edff5
commit 514ffb74aa
2 changed files with 223 additions and 373 deletions

View File

@ -231,383 +231,33 @@ class Webpages extends \Zotlabs\Web\Controller {
$website = dirname($source) . '/' . $tmp_folder_name;
$zip->extractTo($website); // change this to the correct site path
$zip->close();
@unlink($source);
@unlink($source); // delete the compressed file now that the content has been extracted
$hubsites = $this->import_website($website);
require_once('include/import.php');
$elements = [];
$elements['pages'] = scan_webpage_elements($website, 'page');
$elements['layouts'] = scan_webpage_elements($website, 'layout');
$elements['blocks'] = scan_webpage_elements($website, 'block');
$channel = \App::get_channel();
$blocks = $this->import_blocks($channel, $hubsites['blocks']);
$pages = $this->import_pages($channel, $hubsites['pages']);
$layouts = $this->import_layouts($channel, $hubsites['layouts']);
if($blocks || $pages || $layouts) { // Without the if statement, the folder is deleted before the import_blocks function completes.
rrmdir($website);
// Import layout first so that pages that reference new layouts will find
// the mid of layout items in the database
foreach($elements['layouts'] as &$layout) {
$layout = import_webpage_element($layout, $channel, 'layout');
}
}
foreach($elements['pages'] as &$page) {
$page = import_webpage_element($page, $channel, 'page');
}
foreach($elements['blocks'] as &$block) {
$block = import_webpage_element($block, $channel, 'block');
}
// Without the if statement below, the folder is deleted before the import completes.
if($elements) {
rrmdir($website); // Delete the temporary decompressed files
}
}
}
}
private function import_website($path) {
$hubsites = [];
$pages = [];
$blocks = [];
$layouts = [];
// Import pages
$dirtoscan = $path . '/pages/';
if (is_dir($dirtoscan)) {
$dirlist = scandir($dirtoscan);
if ($dirlist) {
foreach ($dirlist as $element) {
if ($element === '.' || $element === '..') {
continue;
}
$folder = $dirtoscan . '/' . $element;
if (is_dir($folder)) {
$jsonfilepath = $folder . '/page.json';
if (is_file($jsonfilepath)) {
$pagejson = json_decode(file_get_contents($jsonfilepath), true);
$pagejson['path'] = $folder . '/' . $pagejson['contentfile'];
if ($pagejson['contentfile'] === '') {
logger('Invalid page content file');
return false;
}
$pagecontent = file_get_contents($folder . '/' . $pagejson['contentfile']);
if (!$pagecontent) {
logger('Failed to get file content for ' . $pagejson['contentfile']);
return false;
}
$pages[] = $pagejson;
}
}
}
}
}
$hubsites['pages'] = $pages;
// Import layouts
$dirtoscan = $path . '/layouts/';
if (is_dir($dirtoscan)) {
$dirlist = scandir($dirtoscan);
if ($dirlist) {
foreach ($dirlist as $element) {
if ($element === '.' || $element === '..') {
continue;
}
$folder = $dirtoscan . '/' . $element;
if (is_dir($folder)) {
$jsonfilepath = $folder . '/layout.json';
if (is_file($jsonfilepath)) {
$layoutjson = json_decode(file_get_contents($jsonfilepath), true);
$layoutjson['path'] = $folder . '/' . $layoutjson['contentfile'];
if ($layoutjson['contentfile'] === '') {
logger('Invalid layout content file');
return false;
}
$layoutcontent = file_get_contents($folder . '/' . $layoutjson['contentfile']);
if (!$layoutcontent) {
logger('Failed to get file content for ' . $layoutjson['contentfile']);
return false;
}
$layouts[] = $layoutjson;
}
}
}
}
}
$hubsites['layouts'] = $layouts;
// Import blocks
$dirtoscan = $path . '/blocks/';
if (is_dir($dirtoscan)) {
$dirlist = scandir($dirtoscan);
if ($dirlist) {
foreach ($dirlist as $element) {
if ($element === '.' || $element === '..') {
continue;
}
$folder = $dirtoscan . '/' . $element;
if (is_dir($folder)) {
$jsonfilepath = $folder . '/block.json';
if (is_file($jsonfilepath)) {
$block = json_decode(file_get_contents($jsonfilepath), true);
$block['path'] = $folder . '/' . $block['contentfile'];
if ($block['contentfile'] === '') {
logger('Invalid block content file');
return false;
}
$blockcontent = file_get_contents($folder . '/' . $block['contentfile']);
if (!$blockcontent) {
logger('Failed to get file content for ' . $block['contentfile']);
return false;
}
$blocks[] = $block;
}
}
}
}
}
$hubsites['blocks'] = $blocks;
return $hubsites;
}
private function import_blocks($channel, $blocks) {
foreach ($blocks as &$b) {
$arr = array();
$arr['item_type'] = ITEM_TYPE_BLOCK;
$namespace = 'BUILDBLOCK';
$arr['uid'] = $channel['channel_id'];
$arr['aid'] = $channel['channel_account_id'];
$iid = q("select iid from iconfig where k = 'BUILDBLOCK' and v = '%s' and cat = 'system'",
dbesc($b['name'])
);
if($iid) {
$iteminfo = q("select mid,created,edited from item where id = %d",
intval($iid[0]['iid'])
);
$arr['mid'] = $arr['parent_mid'] = $iteminfo[0]['mid'];
$arr['created'] = $iteminfo[0]['created'];
$arr['edited'] = (($b['edited']) ? datetime_convert('UTC', 'UTC', $b['edited']) : datetime_convert());
} else {
$arr['created'] = (($b['created']) ? datetime_convert('UTC', 'UTC', $b['created']) : datetime_convert());
$arr['edited'] = datetime_convert('UTC', 'UTC', '0000-00-00 00:00:00');
$arr['mid'] = $arr['parent_mid'] = item_message_id();
}
$arr['title'] = $b['title'];
$arr['body'] = file_get_contents($b['path']);
$arr['owner_xchan'] = get_observer_hash();
$arr['author_xchan'] = (($b['author_xchan']) ? $b['author_xchan'] : get_observer_hash());
if(($b['mimetype'] === 'text/bbcode' || $b['mimetype'] === 'text/html' ||
$b['mimetype'] === 'text/markdown' ||$b['mimetype'] === 'text/plain' ||
$b['mimetype'] === 'application/x-pdl' ||$b['mimetype'] === 'application/x-php')) {
$arr['mimetype'] = $b['mimetype'];
} else {
$arr['mimetype'] = 'text/bbcode';
}
$pagetitle = $b['name'];
// Verify ability to use html or php!!!
$execflag = false;
if ($arr['mimetype'] === 'application/x-php') {
$z = q("select account_id, account_roles, channel_pageflags from account left join channel on channel_account_id = account_id where channel_id = %d limit 1", intval(local_channel())
);
if ($z && (($z[0]['account_roles'] & ACCOUNT_ROLE_ALLOWCODE) || ($z[0]['channel_pageflags'] & PAGE_ALLOWCODE))) {
$execflag = true;
}
}
$remote_id = 0;
$z = q("select * from iconfig where v = '%s' and k = '%s' and cat = 'service' limit 1", dbesc($pagetitle), dbesc($namespace));
$i = q("select id, edited, item_deleted from item where mid = '%s' and uid = %d limit 1", dbesc($arr['mid']), intval(local_channel())
);
if ($z && $i) {
$remote_id = $z[0]['id'];
$arr['id'] = $i[0]['id'];
// don't update if it has the same timestamp as the original
if ($arr['edited'] > $i[0]['edited'])
$x = item_store_update($arr, $execflag);
} else {
if (($i) && (intval($i[0]['item_deleted']))) {
// was partially deleted already, finish it off
q("delete from item where mid = '%s' and uid = %d", dbesc($arr['mid']), intval(local_channel())
);
}
$x = item_store($arr, $execflag);
}
if ($x['success']) {
$item_id = $x['item_id'];
update_remote_id($channel, $item_id, $arr['item_type'], $pagetitle, $namespace, $remote_id, $arr['mid']);
$b['import_success'] = 1;
} else {
$b['import_success'] = 0;
}
}
return $blocks;
}
private function import_pages($channel, $pages) {
foreach ($pages as &$p) {
$arr = array();
$arr['item_type'] = ITEM_TYPE_WEBPAGE;
$namespace = 'WEBPAGE';
$arr['uid'] = $channel['channel_id'];
$arr['aid'] = $channel['channel_account_id'];
if($p['pagelink']) {
require_once('library/urlify/URLify.php');
$pagetitle = strtolower(\URLify::transliterate($p['pagelink']));
}
$arr['layout_mid'] = ''; // by default there is no layout associated with the page
// If a layout was specified, find it in the database and get its info. If
// it does not exist, leave layout_mid empty
logger('hubsites plugin: $p[layout] = ' . $p['layout']);
if($p['layout'] !== '') {
$liid = q("select iid from iconfig where k = 'PDL' and v = '%s' and cat = 'system'",
dbesc($p['layout'])
);
if($liid) {
$linfo = q("select mid from item where id = %d",
intval($liid[0]['iid'])
);
logger('hubsites plugin: $linfo= ' . json_encode($linfo,true));
$arr['layout_mid'] = $linfo[0]['mid'];
}
}
// See if the page already exists
$iid = q("select iid from iconfig where k = 'WEBPAGE' and v = '%s' and cat = 'system'",
dbesc($pagetitle)
);
if($iid) {
// Get the existing page info
$pageinfo = q("select mid,layout_mid,created,edited from item where id = %d",
intval($iid[0]['iid'])
);
$arr['mid'] = $arr['parent_mid'] = $pageinfo[0]['mid'];
$arr['created'] = $pageinfo[0]['created'];
$arr['edited'] = (($p['edited']) ? datetime_convert('UTC', 'UTC', $p['edited']) : datetime_convert());
} else {
$arr['created'] = (($p['created']) ? datetime_convert('UTC', 'UTC', $p['created']) : datetime_convert());
$arr['edited'] = datetime_convert('UTC', 'UTC', '0000-00-00 00:00:00');
$arr['mid'] = $arr['parent_mid'] = item_message_id();
}
$arr['title'] = $p['title'];
$arr['body'] = file_get_contents($p['path']);
$arr['term'] = $p['term']; // Not sure what this is supposed to be
$arr['owner_xchan'] = get_observer_hash();
$arr['author_xchan'] = (($p['author_xchan']) ? $p['author_xchan'] : get_observer_hash());
if(($p['mimetype'] === 'text/bbcode' || $p['mimetype'] === 'text/html' ||
$p['mimetype'] === 'text/markdown' ||$p['mimetype'] === 'text/plain' ||
$p['mimetype'] === 'application/x-pdl' ||$p['mimetype'] === 'application/x-php')) {
$arr['mimetype'] = $p['mimetype'];
} else {
$arr['mimetype'] = 'text/bbcode';
}
// Verify ability to use html or php!!!
$execflag = false;
if ($arr['mimetype'] === 'application/x-php') {
$z = q("select account_id, account_roles, channel_pageflags from account left join channel on channel_account_id = account_id where channel_id = %d limit 1", intval(local_channel())
);
if ($z && (($z[0]['account_roles'] & ACCOUNT_ROLE_ALLOWCODE) || ($z[0]['channel_pageflags'] & PAGE_ALLOWCODE))) {
$execflag = true;
}
}
$remote_id = 0;
$z = q("select * from iconfig where v = '%s' and k = '%s' and cat = 'system' limit 1",
dbesc($pagetitle),
dbesc($namespace)
);
$i = q("select id, edited, item_deleted from item where mid = '%s' and uid = %d limit 1",
dbesc($arr['mid']),
intval(local_channel())
);
if ($z && $i) {
$remote_id = $z[0]['id'];
$arr['id'] = $i[0]['id'];
// don't update if it has the same timestamp as the original
if ($arr['edited'] > $i[0]['edited'])
$x = item_store_update($arr, $execflag);
} else {
if (($i) && (intval($i[0]['item_deleted']))) {
// was partially deleted already, finish it off
q("delete from item where mid = '%s' and uid = %d", dbesc($arr['mid']), intval(local_channel())
);
}
logger('hubsites plugin: item_store= ' . json_encode($arr,true));
$x = item_store($arr, $execflag);
}
if ($x['success']) {
$item_id = $x['item_id'];
update_remote_id($channel, $item_id, $arr['item_type'], $pagetitle, $namespace, $remote_id, $arr['mid']);
$p['import_success'] = 1;
} else {
$p['import_success'] = 0;
}
}
return $pages;
}
private function import_layouts($channel, $layouts) {
foreach ($layouts as &$p) {
$arr = array();
$arr['item_type'] = ITEM_TYPE_PDL;
$namespace = 'PDL';
$arr['uid'] = $channel['channel_id'];
$arr['aid'] = $channel['channel_account_id'];
$pagetitle = $p['name'];
// See if the layout already exists
$iid = q("select iid from iconfig where k = 'PDL' and v = '%s' and cat = 'system'",
dbesc($pagetitle)
);
if($iid) {
// Get the existing layout info
$info = q("select mid,layout_mid,created,edited from item where id = %d",
intval($iid[0]['iid'])
);
$arr['mid'] = $arr['parent_mid'] = $info[0]['mid'];
$arr['created'] = $info[0]['created'];
$arr['edited'] = (($p['edited']) ? datetime_convert('UTC', 'UTC', $p['edited']) : datetime_convert());
} else {
$arr['created'] = (($p['created']) ? datetime_convert('UTC', 'UTC', $p['created']) : datetime_convert());
$arr['edited'] = datetime_convert('UTC', 'UTC', '0000-00-00 00:00:00');
$arr['mid'] = $arr['parent_mid'] = item_message_id();
}
$arr['title'] = $p['description'];
$arr['body'] = file_get_contents($p['path']);
$arr['term'] = $p['term']; // Not sure what this is supposed to be
$arr['owner_xchan'] = get_observer_hash();
$arr['author_xchan'] = (($p['author_xchan']) ? $p['author_xchan'] : get_observer_hash());
$arr['mimetype'] = 'text/bbcode';
$remote_id = 0;
$z = q("select * from iconfig where v = '%s' and k = '%s' and cat = 'system' limit 1",
dbesc($pagetitle),
dbesc($namespace)
);
$i = q("select id, edited, item_deleted from item where mid = '%s' and uid = %d limit 1",
dbesc($arr['mid']),
intval(local_channel())
);
if ($z && $i) {
$remote_id = $z[0]['id'];
$arr['id'] = $i[0]['id'];
// don't update if it has the same timestamp as the original
if ($arr['edited'] > $i[0]['edited'])
$x = item_store_update($arr, $execflag);
} else {
if (($i) && (intval($i[0]['item_deleted']))) {
// was partially deleted already, finish it off
q("delete from item where mid = '%s' and uid = %d", dbesc($arr['mid']), intval(local_channel())
);
}
$x = item_store($arr, $execflag);
}
if ($x['success']) {
$item_id = $x['item_id'];
update_remote_id($channel, $item_id, $arr['item_type'], $pagetitle, $namespace, $remote_id, $arr['mid']);
$p['import_success'] = 1;
} else {
$p['import_success'] = 0;
}
}
return $layouts;
}
}

View File

@ -1217,3 +1217,203 @@ function convert_oldfields(&$arr,$old,$new) {
unset($arr[$old]);
}
}
function scan_webpage_elements($path, $type) {
$dirtoscan = $path;
switch ($type) {
case 'page':
$dirtoscan .= '/pages/';
$json_filename = 'page.json';
break;
case 'layout':
$dirtoscan .= '/layouts/';
$json_filename = 'layout.json';
break;
case 'block':
$dirtoscan .= '/blocks/';
$json_filename = 'block.json';
break;
default :
return array();
}
$elements = [];
if (is_dir($dirtoscan)) {
$dirlist = scandir($dirtoscan);
if ($dirlist) {
foreach ($dirlist as $element) {
if ($element === '.' || $element === '..') {
continue;
}
$folder = $dirtoscan . '/' . $element;
if (is_dir($folder)) {
$jsonfilepath = $folder . '/' . $json_filename;
if (is_file($jsonfilepath)) {
$metadata = json_decode(file_get_contents($jsonfilepath), true);
$metadata['path'] = $folder . '/' . $metadata['contentfile'];
if ($metadata['contentfile'] === '') {
logger('Invalid ' . $type . ' content file');
return false;
}
$content = file_get_contents($folder . '/' . $metadata['contentfile']);
if (!$content) {
logger('Failed to get file content for ' . $metadata['contentfile']);
return false;
}
$elements[] = $metadata;
}
}
}
}
}
return $elements;
}
function import_webpage_element($element, $channel, $type) {
$arr = array(); // construct information for the webpage element item table record
switch ($type) {
//
// PAGES
//
case 'page':
$arr['item_type'] = ITEM_TYPE_WEBPAGE;
$namespace = 'WEBPAGE';
$name = $element['pagelink'];
if($name) {
require_once('library/urlify/URLify.php');
$name = strtolower(\URLify::transliterate($name));
}
$arr['title'] = $element['title'];
$arr['term'] = $element['term'];
$arr['layout_mid'] = ''; // by default there is no layout associated with the page
// If a layout was specified, find it in the database and get its info. If
// it does not exist, leave layout_mid empty
if($element['layout'] !== '') {
$liid = q("select iid from iconfig where k = 'PDL' and v = '%s' and cat = 'system'",
dbesc($element['layout'])
);
if($liid) {
$linfo = q("select mid from item where id = %d",
intval($liid[0]['iid'])
);
$arr['layout_mid'] = $linfo[0]['mid'];
}
}
break;
//
// LAYOUTS
//
case 'layout':
$arr['item_type'] = ITEM_TYPE_PDL;
$namespace = 'PDL';
$name = $element['name'];
$arr['title'] = $element['description'];
$arr['term'] = $element['term'];
break;
//
// BLOCKS
//
case 'block':
$arr['item_type'] = ITEM_TYPE_BLOCK;
$namespace = 'BUILDBLOCK';
$name = $element['name'];
$arr['title'] = $element['title'];
break;
default :
return null; // return null if invalid element type
}
$arr['uid'] = $channel['channel_id'];
$arr['aid'] = $channel['channel_account_id'];
// Check if an item already exists based on the name
$iid = q("select iid from iconfig where k = '" . $namespace . "' and v = '%s' and cat = 'system'",
dbesc($name)
);
if($iid) { // If the item does exist, get the item metadata
$iteminfo = q("select mid,created,edited from item where id = %d",
intval($iid[0]['iid'])
);
$arr['mid'] = $arr['parent_mid'] = $iteminfo[0]['mid'];
$arr['created'] = $iteminfo[0]['created'];
$arr['edited'] = (($element['edited']) ? datetime_convert('UTC', 'UTC', $element['edited']) : datetime_convert());
} else { // otherwise, generate the creation times and unique id
$arr['created'] = (($element['created']) ? datetime_convert('UTC', 'UTC', $element['created']) : datetime_convert());
$arr['edited'] = datetime_convert('UTC', 'UTC', '0000-00-00 00:00:00');
$arr['mid'] = $arr['parent_mid'] = item_message_id();
}
// Import the actual element content
$arr['body'] = file_get_contents($element['path']);
// The element owner is the channel importing the elements
$arr['owner_xchan'] = get_observer_hash();
// The author is either the owner or whomever was specified
$arr['author_xchan'] = (($element['author_xchan']) ? $element['author_xchan'] : get_observer_hash());
// Import mimetype if it is a valid mimetype for the element
$mimetypes = [ 'text/bbcode',
'text/html',
'text/markdown',
'text/plain',
'application/x-pdl',
'application/x-php'
];
// Blocks and pages can have any mimetype, but layouts must be text/bbcode
if((in_array($element['mimetype'], $mimetypes)) && ($type === 'page' || $type === 'block') ) {
$arr['mimetype'] = $element['mimetype'];
} else {
$arr['mimetype'] = 'text/bbcode';
}
// Verify ability to use html or php!!!
$execflag = false;
if ($arr['mimetype'] === 'application/x-php') {
$z = q("select account_id, account_roles, channel_pageflags from account "
. "left join channel on channel_account_id = account_id where channel_id = %d limit 1",
intval(local_channel())
);
if ($z && (($z[0]['account_roles'] & ACCOUNT_ROLE_ALLOWCODE) || ($z[0]['channel_pageflags'] & PAGE_ALLOWCODE))) {
$execflag = true;
}
}
$z = q("select * from iconfig where v = '%s' and k = '%s' and cat = 'service' limit 1",
dbesc($name),
dbesc($namespace)
);
$i = q("select id, edited, item_deleted from item where mid = '%s' and uid = %d limit 1",
dbesc($arr['mid']),
intval(local_channel())
);
$remote_id = 0;
if ($z && $i) {
$remote_id = $z[0]['id'];
$arr['id'] = $i[0]['id'];
// don't update if it has the same timestamp as the original
if ($arr['edited'] > $i[0]['edited'])
$x = item_store_update($arr, $execflag);
} else {
if (($i) && (intval($i[0]['item_deleted']))) {
// was partially deleted already, finish it off
q("delete from item where mid = '%s' and uid = %d",
dbesc($arr['mid']),
intval(local_channel())
);
}
$x = item_store($arr, $execflag);
}
if ($x['success']) {
$item_id = $x['item_id'];
update_remote_id($channel, $item_id, $arr['item_type'], $name, $namespace, $remote_id, $arr['mid']);
$element['import_success'] = 1;
} else {
$element['import_success'] = 0;
}
return $element;
}