What this checkin does is catch the case where a comment arrived and there's a missing top-level post to match it with. So we'll send a request back to the sender that you've never seen this thread and please send a fresh copy of the entire conversation to date. We could soon have posts in the matrix from different platforms from days gone by, which have been migrated into the modern world. We'll be polite and not deliver these to everybody. However, if someone comments on one of these antique threads we wouldn't be able to see it in our own matrix because we won't have a copy of the parent post. So this rectifies that situation. Be aware that item deletion may need to change to keep "hard deleted" items indefinitely so that they don't keep coming back. We'll have to null out the important data of the former item to accomplish the deletion aspect.

include/deliver.php

@@ -38,15 +38,30 @@ function deliver_run($argv, $argc) {
 				continue;
 			}
 
-			if($r[0]['outq_posturl'] === z_root() . '/post') {
+			$notify = json_decode($r[0]['outq_notify'],true);
+
+			$sendtoweb = false;
+			if(array_key_exists('iv',$notify) && (! $r[0]['outq_msg']))
+				$sendtoweb = true;
+
+			if(($r[0]['outq_posturl'] === z_root() . '/post') && (! $sendtoweb)) {
 				logger('deliver: local delivery', LOGGER_DEBUG);
 				// local delivery
 				// we should probably batch these and save a few delivery processes
 				// If there is no outq_msg, this is a refresh_all message which does not require local handling
+				// also send 'request' packets to the webservice so it can decode the packet
 				if($r[0]['outq_msg']) {
-					$msg = array('body' => json_encode(array('pickup' => array(array('notify' => json_decode($r[0]['outq_notify'],true),'message' => json_decode($r[0]['outq_msg'],true))))));
+					$m = json_decode($r[0]['outq_msg'],true);
-
+					if(array_key_exists('message_list',$m)) {
-					zot_import($msg,z_root());
+						foreach($m['message_list'] as $mm) {
+							$msg = array('body' => json_encode(array('pickup' => array(array('notify' => $notify,'message' => $mm)))));
+							zot_import($msg,z_root());
+						}
+					}	
+					else {	
+						$msg = array('body' => json_encode(array('pickup' => array(array('notify' => $notify,'message' => $m)))));
+						zot_import($msg,z_root());
+					}
 					$r = q("delete from outq where outq_hash = '%s' limit 1",
 						dbesc($argv[$x])
 					);

include/items.php

@@ -4249,14 +4249,24 @@ function fetch_post_tags($items,$link = false) {
 
 
 
-function zot_feed($uid,$observer_xchan,$mindate) {
+function zot_feed($uid,$observer_xchan,$arr) {
 
 
 	$result = array();
-	$mindate = datetime_convert('UTC','UTC',$mindate);
+	$mindate = null;
+	$message_id = null;
+
+	if(array_key_exists('mindate',$arr)) {
+		$mindate = datetime_convert('UTC','UTC',$arr['mindate']);
+	}
+
+	if(array_key_exists('message_id',$arr)) {
+		$message_id = $arr['message_id'];
+	}
+
+
 	if(! $mindate)
 		$mindate = NULL_DATE;
-
 	$mindate = dbesc($mindate);
 
 	logger('zot_feed: ' . $uid);
@@ -4278,6 +4288,11 @@ function zot_feed($uid,$observer_xchan,$mindate) {
 	else
 		$limit = " limit 0, 50 ";
 
+	if($message_id) {
+		$sql_extra .= " and mid = '" . dbesc($message_id) . "' ";
+		$limit = '';
+	}
+
 	$items = array();
 
 	if(is_sys_channel($uid)) {

include/notifier.php

@@ -58,6 +58,7 @@ require_once('include/html2plain.php');
  *       expire                 channel_id
  *       relay					item_id (item was relayed to owner, we will deliver it as owner)
  *       location               channel_id
+ *       request                channel_id            xchan_hash             message_id
  *
  */
 
@@ -142,6 +143,7 @@ function notifier_run($argv, $argc){
 
 
 	$expire = false;
+	$request = false;
 	$mail = false;
 	$fsuggest = false;
 	$top_level = false;
@@ -174,6 +176,23 @@ function notifier_run($argv, $argc){
 		if($s)
 			$channel = $s[0];
 
+	}
+	elseif($cmd === 'request') {
+		$channel_id = $item_id;
+		$xchan = $argv[3];
+		$request_message_id = $argv[4];
+
+		$s = q("select * from channel where channel_id = %d limit 1",
+			intval($channel_id)
+		);
+		if($s)
+			$channel = $s[0];
+
+		$private = true;
+		$recipients[] = $xchan;
+		$packet_type = 'request';
+		$normal_mode = false;
+
 	}
 	elseif($cmd === 'expire') {
 
@@ -550,6 +569,7 @@ function notifier_run($argv, $argc){
 				'mail' => $mail,
 				'location' => $location,
 				'fsuggest' => $fsuggest,
+				'request' => $request,
 				'normal_mode' => $normal_mode,
 				'packet_type' => $packet_type,
 				'walltowall' => $walltowall
@@ -579,6 +599,21 @@ function notifier_run($argv, $argc){
 				dbesc('')
 			);
 		}
+		elseif($packet_type === 'request') {
+			$n = zot_build_packet($channel,'request',$env_recips,$hub['hubloc_sitekey'],$hash,array('message_id' => $request_message_id));
+			q("insert into outq ( outq_hash, outq_account, outq_channel, outq_driver, outq_posturl, outq_async, outq_created, outq_updated, outq_notify, outq_msg ) values ( '%s', %d, %d, '%s', '%s', %d, '%s', '%s', '%s', '%s' )",
+				dbesc($hash),
+				intval($channel['channel_account_id']),
+				intval($channel['channel_id']),
+				dbesc('zot'),
+				dbesc($hub['hubloc_callback']),
+				intval(1),
+				dbesc(datetime_convert()),
+				dbesc(datetime_convert()),
+				dbesc($n),
+				dbesc('')
+			);
+		}
 		else {
 			$n = zot_build_packet($channel,'notify',$env_recips,(($private) ? $hub['hubloc_sitekey'] : null),$hash);
 			q("insert into outq ( outq_hash, outq_account, outq_channel, outq_driver, outq_posturl, outq_async, outq_created, outq_updated, outq_notify, outq_msg ) values ( '%s', %d, %d, '%s', '%s', %d, '%s', '%s', '%s', '%s' )",

include/onepoll.php

@@ -15,12 +15,6 @@ function onepoll_run($argv, $argc){
 
 	logger('onepoll: start');
 	
-	$manual_id  = 0;
-	$generation = 0;
-
-	$force      = false;
-	$restart    = false;
-
 	if(($argc > 1) && (intval($argv[1])))
 		$contact_id = intval($argv[1]);
 
@@ -28,7 +22,7 @@ function onepoll_run($argv, $argc){
 		logger('onepoll: no contact');
 		return;
 	}
-	
+
 	$d = datetime_convert();
 
 	$contacts = q("SELECT abook.*, xchan.*, account.*
@@ -90,7 +84,8 @@ function onepoll_run($argv, $argc){
 	$x = zot_refresh($contact,$importer);
 
 	$responded = false;
-	$updated = datetime_convert();
+	$updated   = datetime_convert();
+	$connected = datetime_convert();
 	if(! $x) {
 		// mark for death by not updating abook_connected, this is caught in include/poller.php
 		q("update abook set abook_updated = '%s' where abook_id = %d limit 1",
@@ -101,7 +96,7 @@ function onepoll_run($argv, $argc){
 	else {
 		q("update abook set abook_updated = '%s', abook_connected = '%s' where abook_id = %d limit 1",
 			dbesc($updated),
-			dbesc($updated),
+			dbesc($connected),
 			intval($contact['abook_id'])
 		);
 		$responded = true;
@@ -120,7 +115,9 @@ function onepoll_run($argv, $argc){
 		if($fetch_feed) {
 
 			$feedurl = str_replace('/poco/','/zotfeed/',$contact['xchan_connurl']);		
-			$x = z_fetch_url($feedurl . '?f=&mindate=' . urlencode($last_update));
+			$feedurl .= '?f=&mindate=' . urlencode($last_update);
+
+			$x = z_fetch_url($feedurl);
 
 			logger('feed_update: ' . print_r($x,true), LOGGER_DATA);
 
@@ -143,8 +140,8 @@ function onepoll_run($argv, $argc){
 		}
 	}
 			
-	// fetch some items
+
-	// set last updated timestamp
+	// update the poco details for this connection
 
 	if($contact['xchan_connurl']) {	
 		$r = q("SELECT xlink_id from xlink 

include/zot.php

@@ -104,7 +104,7 @@ function zot_get_hublocs($hash) {
  * @returns string json encoded zot packet
  */
 
-function zot_build_packet($channel,$type = 'notify',$recipients = null, $remote_key = null, $secret = null) {
+function zot_build_packet($channel,$type = 'notify',$recipients = null, $remote_key = null, $secret = null, $extra = null) {
 
 	$data = array(
 		'type' => $type,
@@ -129,6 +129,12 @@ function zot_build_packet($channel,$type = 'notify',$recipients = null, $remote_
 		$data['secret_sig'] = base64url_encode(rsa_sign($secret,$channel['channel_prvkey']));
 	}
 
+	if($extra) {
+		foreach($extra as $k => $v)
+			$data[$k] = $v;
+	}
+
+
 	logger('zot_build_packet: ' . print_r($data,true), LOGGER_DATA);
 
 	// Hush-hush ultra top-secret mode
@@ -1436,6 +1442,13 @@ function process_delivery($sender,$arr,$deliveries,$relay,$public = false) {
 			);
 			if(! $r) {
 				$result[] = array($d['hash'],'comment parent not found',$channel['channel_name'] . ' <' . $channel['channel_address'] . '@' . get_app()->get_hostname() . '>',$arr['mid']);
+
+				// we don't seem to have a copy of this conversation or at least the parent - request a copy of the entire conversation to date.
+				// Don't do this if it's a relay post as we're the ones who are supposed to have the copy and we don't want the request to loop.
+
+				if(! $relay)
+					proc_run('php', 'include/notifier.php', 'request', $channel['channel_id'], $sender['hash'], $arr['parent_mid']);
+
 				continue;
 			}
 			if($relay) {
@@ -2793,3 +2806,71 @@ function import_author_zot($x) {
 	return false;
 }
 
+function zot_process_message_request($data) {
+	$ret = array('success' => false);
+	if(! $data['message_id']) {
+		$ret['message'] = 'no message_id';
+		logger('no message_id');
+		return $ret;
+	}
+
+	$sender = $data['sender'];
+	$sender_hash = make_xchan_hash($sender['guid'],$sender['guid_sig']);
+
+	$arr = $data['recipients'][0];
+	$recip_hash = make_xchan_hash($arr['guid'],$arr['guid_sig']);
+	$c = q("select * from channel left join xchan on channel_hash = xchan_hash where channel_hash = '%s' limit 1",
+		dbesc($recip_hash)
+	);
+	if(! $c) {
+		logger('recipient channel not found.');
+		$ret['message'] .= 'recipient not found.' . EOL;
+		return $ret;
+	}
+	$messages = zot_feed($c[0]['channel_id'],$sender_hash,array('message_id' => $data['message_id']));
+	if($messages) {
+		$env_recips = null;
+
+		$r = q("select hubloc_guid, hubloc_url, hubloc_sitekey, hubloc_network, hubloc_flags, hubloc_callback, hubloc_host 
+			from hubloc where hubloc_hash = '" . dbesc($sender_hash) . "' and not (hubloc_flags & %d)
+			and not (hubloc_status & %d) group by hubloc_sitekey",
+            intval(HUBLOC_FLAGS_DELETED),
+            intval(HUBLOC_OFFLINE)
+        );
+	    if(! $r) {
+	        logger('no hubs');
+    	    return $ret;
+    	}
+		$hubs = $r;
+		$hublist = array();
+		$keys = array();
+
+		$private = ((array_key_exists('flags',$messages[0]) && in_array('private',$messages[0]['flags'])) ? true : false);
+		if($private)
+			$env_recips = array('guid' => $sender['guid'],'guid_sig' => $sender['guid_sig'],'hash' => $sender_hash);
+		$data_packet = json_encode(array('message_list' => $messages));
+		
+		foreach($hubs as $hub) {
+			$hash = random_string();
+			$n = zot_build_packet($c[0],'notify',$env_recips,(($private) ? $hub['hubloc_sitekey'] : null),$hash);
+			q("insert into outq ( outq_hash, outq_account, outq_channel, outq_driver, outq_posturl, outq_async, 
+				outq_created, outq_updated, outq_notify, outq_msg ) 
+				values ( '%s', %d, %d, '%s', '%s', %d, '%s', '%s', '%s', '%s' )",
+				dbesc($hash),
+				intval($c[0]['channel_account_id']),
+				intval($c[0]['channel_id']),
+				dbesc('zot'),
+				dbesc($hub['hubloc_callback']),
+				intval(1),
+				dbesc(datetime_convert()),
+				dbesc(datetime_convert()),
+				dbesc($n),
+				dbesc($data_packet)
+			);
+			proc_run('php','include/deliver.php',$hash);
+		}
+
+	}
+	$ret['success'] = true;
+	return $ret;
+}

mod/post.php

@@ -507,6 +507,7 @@ function post_post(&$a) {
 		json_return_and_die($ret);
 	}
 
+
 	if($msgtype === 'pickup') {
 
 		/**
@@ -597,7 +598,14 @@ function post_post(&$a) {
 			$ret['success'] = true;
 			$ret['pickup'] = array();
 			foreach($r as $rr) {
-				$ret['pickup'][] = array('notify' => json_decode($rr['outq_notify'],true),'message' => json_decode($rr['outq_msg'],true));
+				$x = json_decode($rr['outq_msg'],true);
+
+				if(array_key_exists('message_list',$x)) {
+					foreach($x['message_list'] as $xx)
+						$ret['pickup'][] = array('notify' => json_decode($rr['outq_notify'],true),'message' => $xx);
+				}
+				else
+					$ret['pickup'][] = array('notify' => json_decode($rr['outq_notify'],true),'message' => $x);
 
 				$x = q("delete from outq where outq_hash = '%s' limit 1",
 					dbesc($rr['outq_hash'])
@@ -796,6 +804,12 @@ function post_post(&$a) {
 		json_return_and_die($ret);
 	}
 
+	if($msgtype === 'request') {
+		// request a particular post/conversation by message_id
+		$x = zot_process_message_request($data);
+		json_return_and_die($x);		
+	}
+
 
 	if($msgtype === 'purge') {
 		if($recipients) {

mod/zotfeed.php

@@ -38,7 +38,7 @@ function zotfeed_init(&$a) {
 
 	logger('zotfeed request: ' . $r[0]['channel_name'], LOGGER_DEBUG);
 
-	$result['messages'] = zot_feed($r[0]['channel_id'],$observer['xchan_hash'],$mindate);
+	$result['messages'] = zot_feed($r[0]['channel_id'],$observer['xchan_hash'],array('mindate' => $mindate));
 	$result['success'] = true;
 	json_return_and_die($result);
 

version.inc

@@ -1 +1 @@
-2014-10-29.843
+2014-10-30.844