From 2844f276b9af6154e36706b7fe53438352493d58 Mon Sep 17 00:00:00 2001 From: redmatrix Date: Thu, 24 Sep 2015 16:57:17 -0700 Subject: [PATCH 1/6] Document the additional content export options --- include/event.php | 5 ++++- mod/tasks.php | 2 +- mod/uexport.php | 15 +++++++++++++-- view/tpl/uexport.tpl | 7 +++++++ 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/include/event.php b/include/event.php index f0a806dfa..e303ad232 100644 --- a/include/event.php +++ b/include/event.php @@ -755,12 +755,15 @@ function event_store_item($arr, $event) { } } + + $item_arr = array(); $prefix = ''; // $birthday = false; if($event['type'] === 'birthday') { - $prefix = t('This event has been added to your calendar.'); + if(! is_sys_channel($arr['uid'])) + $prefix = t('This event has been added to your calendar.'); // $birthday = true; // The event is created on your own site by the system, but appears to belong diff --git a/mod/tasks.php b/mod/tasks.php index ed267cc92..b1b81473b 100644 --- a/mod/tasks.php +++ b/mod/tasks.php @@ -73,7 +73,7 @@ function tasks_post(&$a) { if(! $text) return array('success' => false); $event = array(); - $event['aid'] = $channel['channel_account_id']; + $event['account'] = $channel['channel_account_id']; $event['uid'] = $channel['channel_id']; $event['event_xchan'] = $channel['channel_hash']; $event['type'] = 'task'; diff --git a/mod/uexport.php b/mod/uexport.php index df66474f1..ee22383a3 100644 --- a/mod/uexport.php +++ b/mod/uexport.php @@ -41,13 +41,24 @@ function uexport_init(&$a) { } function uexport_content(&$a) { + + $y = datetime_convert('UTC',date_default_timezone_get(),'now','Y'); + + $yearurl = z_root() . '/uexport/' . $y; + $janurl = z_root() . '/uexport/' . $y . '/1'; + $impurl = '/import_items'; $o = replace_macros(get_markup_template('uexport.tpl'), array( '$title' => t('Export Channel'), '$basictitle' => t('Export Channel'), - '$basic' => t('Export your basic channel information to a small file. This acts as a backup of your connections, permissions, profile and basic data, which can be used to import your data to a new hub, but does not contain your content.'), + '$basic' => t('Export your basic channel information to a file. This acts as a backup of your connections, permissions, profile and basic data, which can be used to import your data to a new server hub, but does not contain your content.'), '$fulltitle' => t('Export Content'), - '$full' => t('Export your channel information and all the content to a JSON backup. This backs up all of your connections, permissions, profile data and the last year of posts. This file may be VERY large. Please be patient - it may take several minutes for this download to begin.'), + '$full' => t('Export your channel information and recent content to a JSON backup that can be restored or imported to another server hub. This backs up all of your connections, permissions, profile data and several months of posts. This file may be VERY large. Please be patient - it may take several minutes for this download to begin.'), '$by_year' => t('Export your posts from a given year.'), + + '$extra' => t('You may also export your posts and conversations for a particular year or month. Adjust the date in your browser location bar to select other dates. If the export fails (possibly due to memory exhaustion on your server hub), please try again selecting a more limited date range.'), + '$extra2' => sprintf( t('To select all posts for a given year, such as this year, visit %2$s'),$yearurl,$yearurl), + '$extra3' => sprintf( t('To select all posts for a given month, such as January of this year, visit %2$s'),$janurl,$janurl), + '$extra4' => sprintf( t('These content files may be imported or restored by visiting %2$s on any site containing your channel. For best results please import or restore these in date order (oldest first).'),$impurl,$impurl) )); return $o; diff --git a/view/tpl/uexport.tpl b/view/tpl/uexport.tpl index b9a1b9572..7eafd97a6 100644 --- a/view/tpl/uexport.tpl +++ b/view/tpl/uexport.tpl @@ -8,5 +8,12 @@

{{$fulltitle}}

{{$full}}

+ +

{{$extra}}

+

{{$extra2}}

+

{{$extra3}}

+ +

{{$extra4}}

+ From 2c665e5e4cbd5530bc979c99022a2513c67a5425 Mon Sep 17 00:00:00 2001 From: redmatrix Date: Thu, 24 Sep 2015 17:41:05 -0700 Subject: [PATCH 2/6] add Diaspora (actually any external service which uses the queue) to delivery reports --- include/deliver.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/deliver.php b/include/deliver.php index 7a43e3d47..82a1ac6df 100644 --- a/include/deliver.php +++ b/include/deliver.php @@ -62,9 +62,17 @@ function deliver_run($argv, $argc) { $result = z_post_url($r[0]['outq_posturl'],$r[0]['outq_msg']); if($result['success'] && $result['return_code'] < 300) { logger('deliver: queue post success to ' . $r[0]['outq_posturl'], LOGGER_DEBUG); + + q("update dreport set status = '%s', dreport_time = '%s' where dreport_queue = '%s' limit 1", + dbesc('accepted for delivery'), + dbesc(datetime_convert()), + dbesc($argv[$x]) + ); + $y = q("delete from outq where outq_hash = '%s'", dbesc($argv[$x]) ); + } else { logger('deliver: queue post returned ' . $result['return_code'] . ' from ' . $r[0]['outq_posturl'],LOGGER_DEBUG); From e558ddae3f0e9cae8800caf878a07f5530d2e31f Mon Sep 17 00:00:00 2001 From: redmatrix Date: Thu, 24 Sep 2015 18:19:33 -0700 Subject: [PATCH 3/6] clean up the delivery reports to make them manageable, more approachable, and translateable --- mod/dreport.php | 52 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/mod/dreport.php b/mod/dreport.php index a20e17372..31a6274c8 100644 --- a/mod/dreport.php +++ b/mod/dreport.php @@ -30,9 +30,48 @@ function dreport_content(&$a) { $o .= '

' . sprintf( t('Delivery report for %1$s'),substr($mid,0,32)) . '...' . '

'; $o .= ''; + for($x = 0; $x < count($r); $x++ ) { + $r[$x]['name'] = escape_tags(substr($r[$x]['dreport_recip'],strpos($r[$x]['dreport_recip'],' '))); + + switch($r[$x]['dreport_result']) { + case 'channel sync processed': + $r[$x]['gravity'] = 0; + $r[$x]['dreport_result'] = t('channel sync processed'); + break; + case 'queued': + $r[$x]['gravity'] = 2; + $r[$x]['dreport_result'] = t('queued'); + break; + case 'posted': + $r[$x]['gravity'] = 3; + $r[$x]['dreport_result'] = t('posted'); + break; + case 'accepted for delivery': + $r[$x]['gravity'] = 4; + $r[$x]['dreport_result'] = t('accepted for delivery'); + break; + case 'updated': + $r[$x]['gravity'] = 5; + $r[$x]['dreport_result'] = t('updated'); + case 'update ignored': + $r[$x]['gravity'] = 6; + $r[$x]['dreport_result'] = t('update ignored'); + break; + case 'permission denied': + $r[$x]['dreport_result'] = t('permission denied'); + $r[$x]['gravity'] = 6; + break; + default: + $r[$x]['gravity'] = 1; + break; + } + } + + usort($r,'dreport_gravity_sort'); + + foreach($r as $rr) { - $name = escape_tags(substr($rr['dreport_recip'],strpos($rr['dreport_recip'],' '))); - $o .= ''; + $o .= ''; } $o .= '
' . $name . '' . escape_tags($rr['dreport_result']) . '' . escape_tags($rr['dreport_time']) . '
' . $rr['name'] . '' . escape_tags($rr['dreport_result']) . '' . escape_tags($rr['dreport_time']) . '
'; @@ -40,4 +79,13 @@ function dreport_content(&$a) { +} + +function dreport_gravity_sort($a,$b) { + if($a['gravity'] == $b['gravity']) { + if($a['name'] === $b['name']) + return strcmp($a['dreport_time'],$b['dreport_time']); + return strcmp($a['name'],$b['name']); + } + return (($a['gravity'] > $b['gravity']) ? 1 : (-1)); } \ No newline at end of file From 98f656cfcbaf81c001802dacd593ff9548345014 Mon Sep 17 00:00:00 2001 From: redmatrix Date: Thu, 24 Sep 2015 22:51:47 -0700 Subject: [PATCH 4/6] delivery optimisations --- include/deliver_hooks.php | 29 +++++++++++++++++++++++++++++ include/notifier.php | 33 +++++++++++++++++++++++++-------- 2 files changed, 54 insertions(+), 8 deletions(-) create mode 100644 include/deliver_hooks.php diff --git a/include/deliver_hooks.php b/include/deliver_hooks.php new file mode 100644 index 000000000..f0d6ba1b1 --- /dev/null +++ b/include/deliver_hooks.php @@ -0,0 +1,29 @@ += $deliveries_per_process) { proc_run('php','include/deliver.php',$deliver); @@ -684,18 +684,35 @@ function notifier_run($argv, $argc){ @time_sleep_until(microtime(true) + (float) $interval); } } + + if($normal_mode) { + $x = q("select * from hook where hook = 'notifier_normal'"); + if($x) + proc_run('php','deliver_hooks.php', $target_item['id']); + } + + if($deliveries) { + $deliver = array(); + + foreach($deliveries as $d) { + + $deliver[] = $d; + + if(count($deliver) >= $deliveries_per_process) { + proc_run('php','include/deliver.php',$deliver); + $deliver = array(); + if($interval) + @time_sleep_until(microtime(true) + (float) $interval); + } + } + } // catch any stragglers - if(count($deliver)) { + if($deliver) proc_run('php','include/deliver.php',$deliver); - } logger('notifier: basic loop complete.', LOGGER_DEBUG); - - if($normal_mode) - call_hooks('notifier_normal',$target_item); - call_hooks('notifier_end',$target_item); From 8aec437c2314758dfaedef190ecaf74245cfc9e7 Mon Sep 17 00:00:00 2001 From: redmatrix Date: Thu, 24 Sep 2015 23:02:41 -0700 Subject: [PATCH 5/6] remove redundant code --- include/notifier.php | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/include/notifier.php b/include/notifier.php index 4ce0c7c9f..3b29229cf 100644 --- a/include/notifier.php +++ b/include/notifier.php @@ -675,14 +675,8 @@ function notifier_run($argv, $argc){ ); } } - $deliveries[] = $hash; - if(count($deliver) >= $deliveries_per_process) { - proc_run('php','include/deliver.php',$deliver); - $deliver = array(); - if($interval) - @time_sleep_until(microtime(true) + (float) $interval); - } + $deliveries[] = $hash; } if($normal_mode) { From ba94f824b992f9ed787a71c2956388e975058ca8 Mon Sep 17 00:00:00 2001 From: redmatrix Date: Fri, 25 Sep 2015 01:31:58 -0700 Subject: [PATCH 6/6] deliveries per process ui for setting --- mod/admin.php | 3 +++ view/tpl/admin_site.tpl | 1 + 2 files changed, 4 insertions(+) diff --git a/mod/admin.php b/mod/admin.php index 2b7bb007d..7d7c50dcf 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -254,6 +254,7 @@ function admin_page_site_post(&$a){ $proxy = ((x($_POST,'proxy')) ? notags(trim($_POST['proxy'])) : ''); $timeout = ((x($_POST,'timeout')) ? intval(trim($_POST['timeout'])) : 60); $delivery_interval = ((x($_POST,'delivery_interval'))? intval(trim($_POST['delivery_interval'])) : 0); + $delivery_batch_count = ((x($_POST,'delivery_batch_count') && $_POST['delivery_batch_count'] > 0)? intval(trim($_POST['delivery_batch_count'])) : 1); $poll_interval = ((x($_POST,'poll_interval')) ? intval(trim($_POST['poll_interval'])) : 0); $maxloadavg = ((x($_POST,'maxloadavg')) ? intval(trim($_POST['maxloadavg'])) : 50); $feed_contacts = ((x($_POST,'feed_contacts')) ? intval($_POST['feed_contacts']) : 0); @@ -261,6 +262,7 @@ function admin_page_site_post(&$a){ set_config('system', 'feed_contacts', $feed_contacts); set_config('system', 'delivery_interval', $delivery_interval); + set_config('system', 'delivery_batch_count', $delivery_batch_count); set_config('system', 'poll_interval', $poll_interval); set_config('system', 'maxloadavg', $maxloadavg); set_config('system', 'frontpage', $frontpage); @@ -442,6 +444,7 @@ function admin_page_site(&$a) { '$proxy' => array('proxy', t("Proxy URL"), get_config('system','proxy'), ""), '$timeout' => array('timeout', t("Network timeout"), (x(get_config('system','curl_timeout'))?get_config('system','curl_timeout'):60), t("Value is in seconds. Set to 0 for unlimited (not recommended).")), '$delivery_interval' => array('delivery_interval', t("Delivery interval"), (x(get_config('system','delivery_interval'))?get_config('system','delivery_interval'):2), t("Delay background delivery processes by this many seconds to reduce system load. Recommend: 4-5 for shared hosts, 2-3 for virtual private servers. 0-1 for large dedicated servers.")), + '$delivery_batch_count' => array('delivery_batch_count', t('Deliveries per process'),(x(get_config('system','delivery_batch_count'))?get_config('system','delivery_batch_count'):1), t("Number of deliveries to attempt in a single operating system process. Adjust if necessary to tune system performance. Recommend: 1-5.")), '$poll_interval' => array('poll_interval', t("Poll interval"), (x(get_config('system','poll_interval'))?get_config('system','poll_interval'):2), t("Delay background polling processes by this many seconds to reduce system load. If 0, use delivery interval.")), '$maxloadavg' => array('maxloadavg', t("Maximum Load Average"), ((intval(get_config('system','maxloadavg')) > 0)?get_config('system','maxloadavg'):50), t("Maximum system load before delivery and poll processes are deferred - default 50.")), '$default_expire_days' => array('default_expire_days', t('Expiration period in days for imported (matrix/network) content'), intval(get_config('system','default_expire_days')), t('0 for no expiration of imported content')), diff --git a/view/tpl/admin_site.tpl b/view/tpl/admin_site.tpl index e2ce3c552..26434b20a 100755 --- a/view/tpl/admin_site.tpl +++ b/view/tpl/admin_site.tpl @@ -78,6 +78,7 @@ {{include file="field_input.tpl" field=$proxyuser}} {{include file="field_input.tpl" field=$timeout}} {{include file="field_input.tpl" field=$delivery_interval}} + {{include file="field_input.tpl" field=$delivery_batch_count}} {{include file="field_input.tpl" field=$poll_interval}} {{include file="field_input.tpl" field=$maxloadavg}} {{include file="field_input.tpl" field=$abandon_days}}