directory sync - this will either work, or it won't work, or it will possibly recurse and blow up the matrix. Hard to say. Do you feel lucky? Well do ya' ... punk? Rule #1 - don't mess with anything unless it's blowing up the matrix. If it doesn't blow up the matrix, but doesn't work, just let it go and let's figure out what it is doing and what it isn't doing.
The flow is as follows: Once a day go out to all the directory servers besides yourself and grab a list of updates. This happens in the poller. If we've never seen them before add them to the updates table. The poller also looks to see if we're a directory server and have updates that haven't yet been processed. It calls onedirsync.php to process each one. If we contact the channel to update and don't find anything (we're just doing a basic zot_finger), set a ud_last timestamp. If this is set we will only try once a day for seven days. Then we stop trying to update. This will probably cause a spike the first time through because you haven't seen any updates before, but we spread out the load over your delivery interval.
This commit is contained in:
friendica
parent
e0391de906
commit
e992cfeca9
11
boot.php
11
boot.php
@ -45,7 +45,7 @@ define ( 'RED_PLATFORM', 'Red Matrix' );
|
||||
define ( 'RED_VERSION', trim(file_get_contents('version.inc')) . 'R');
|
||||
define ( 'ZOT_REVISION', 1 );
|
||||
|
||||
define ( 'DB_UPDATE_VERSION', 1074 );
|
||||
define ( 'DB_UPDATE_VERSION', 1075 );
|
||||
|
||||
define ( 'EOL', '<br />' . "\r\n" );
|
||||
define ( 'ATOM_TIME', 'Y-m-d\TH:i:s\Z' );
|
||||
@ -296,6 +296,13 @@ define ( 'POLL_MULTIPLE_CHOICE', 0x0004);
|
||||
define ( 'POLL_OVERWRITE', 0x8000); // If you vote twice remove the prior entry
|
||||
|
||||
|
||||
|
||||
define ( 'UPDATE_FLAGS_UPDATED', 0x0001);
|
||||
define ( 'UPDATE_FLAGS_DELETED', 0x1000);
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Maximum number of "people who like (or don't like) this" that we will list by name
|
||||
*/
|
||||
@ -344,7 +351,7 @@ define ( 'XCHAN_FLAGS_HIDDEN', 0x0001);
|
||||
define ( 'XCHAN_FLAGS_ORPHAN', 0x0002);
|
||||
define ( 'XCHAN_FLAGS_CENSORED', 0x0004);
|
||||
define ( 'XCHAN_FLAGS_SELFCENSORED', 0x0008);
|
||||
|
||||
define ( 'XCHAN_FLAGS_DELETED', 0x1000);
|
||||
/*
|
||||
* Traficlights for Administration of HubLoc
|
||||
* to detect problems in inter server communication
|
||||
|
||||
@ -13,7 +13,7 @@ function sync_directories($dirmode) {
|
||||
return;
|
||||
|
||||
$r = q("select * from site where (site_flags & %d) and site_url != '%s'",
|
||||
intval(DIRECTORY_MODE_PRIMARY),
|
||||
intval(DIRECTORY_MODE_PRIMARY|DIRECTORY_MODE_SECONDARY),
|
||||
dbesc(z_root())
|
||||
);
|
||||
|
||||
@ -34,16 +34,74 @@ function sync_directories($dirmode) {
|
||||
dbesc($r[0]['site_directory'])
|
||||
);
|
||||
|
||||
$r = q("select * from site where (site_flags & %d) and site_url != '%s'",
|
||||
intval(DIRECTORY_MODE_PRIMARY|DIRECTORY_MODE_SECONDARY),
|
||||
dbesc(z_root())
|
||||
);
|
||||
|
||||
}
|
||||
if(! $r)
|
||||
return;
|
||||
|
||||
foreach($r as $rr) {
|
||||
if(! $rr['site_directory'])
|
||||
continue;
|
||||
$x = z_fetch_url($rr['site_directory'] . '?f=&sync=' . urlencode($rr['site_sync']));
|
||||
if(! $x['success'])
|
||||
continue;
|
||||
$j = json_decode($x['body'],true);
|
||||
if((! $j['transactions']) || (! is_array($j['transactions'])))
|
||||
continue;
|
||||
|
||||
q("update site set site_sync = '%s' where site_url = '%s' limit 1",
|
||||
dbesc(datetime_convert()),
|
||||
dbesc($rr['site_url'])
|
||||
);
|
||||
|
||||
logger('sync_directories: ' . $rr['site_url'] . ': ' . print_r($j,true), LOGGER_DATA);
|
||||
|
||||
|
||||
|
||||
if(count($j['transactions'])) {
|
||||
foreach($j['transactions'] as $t) {
|
||||
$r = q("select * from updates where ud_guid = '%s' limit 1",
|
||||
dbesc($t['transaction_id'])
|
||||
);
|
||||
if($r)
|
||||
continue;
|
||||
$ud_flags = 0;
|
||||
if(is_array($t['flags']) && in_array('deleted',$t['flags']))
|
||||
$ud_flags |= UPDATE_FLAGS_DELETED;
|
||||
$z = q("insert into updates ( ud_hash, ud_guid, ud_date, ud_flags, ud_addr )
|
||||
values ( '%s', '%s', '%s', '%d, '%s' ) ",
|
||||
dbesc($t['hash']),
|
||||
dbesc($t['transaction_id']),
|
||||
dbesc($t['timestamp']),
|
||||
intval($ud_flags),
|
||||
dbesc($t['address'])
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function update_directory_entry($ud) {
|
||||
|
||||
logger('update_directory_entry: ' . print_r($ud,true), LOGGER_DATA);
|
||||
|
||||
if($ud['ud_addr'] && (! ($ud['ud_flags'] & UPDATE_FLAGS_DELETED))) {
|
||||
$x = zot_finger($ud['ud_addr'],'');
|
||||
if($x['success']) {
|
||||
$j = json_decode($x['body'],true);
|
||||
$y = import_xchan($j,0);
|
||||
}
|
||||
else {
|
||||
$r = q("update updates set ud_last = '%s' where ud_addr = '%s'",
|
||||
dbesc(datetime_convert()),
|
||||
dbesc($ud['ud_addr'])
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
42
include/onedirsync.php
Normal file
42
include/onedirsync.php
Normal file
@ -0,0 +1,42 @@
|
||||
<?php /** @file */
|
||||
|
||||
require_once('boot.php');
|
||||
require_once('include/cli_startup.php');
|
||||
require_once('include/zot.php');
|
||||
require_once('include/dir_fns.php');
|
||||
|
||||
|
||||
function onedirsync_run($argv, $argc){
|
||||
|
||||
|
||||
cli_startup();
|
||||
$a = get_app();
|
||||
|
||||
logger('onedirsync: start');
|
||||
|
||||
if(($argc > 1) && (intval($argv[1])))
|
||||
$update_id = intval($argv[1]);
|
||||
|
||||
if(! $update_id) {
|
||||
logger('onedirsync: no update');
|
||||
return;
|
||||
}
|
||||
|
||||
$r = q("select * from updates where ud_id = %d limit 1",
|
||||
intval($update_id)
|
||||
);
|
||||
|
||||
if(! $r)
|
||||
return;
|
||||
if($r['ud_flags'] & UPDATE_FLAGS_UPDATED)
|
||||
return;
|
||||
|
||||
update_directory_entry($r[0]);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (array_search(__file__,get_included_files())===0){
|
||||
onedirsync_run($argv,$argc);
|
||||
killme();
|
||||
}
|
||||
@ -69,6 +69,8 @@ function poller_run($argv, $argc){
|
||||
$d1 = get_config('system','last_expire_day');
|
||||
$d2 = intval(datetime_convert('UTC','UTC','now','d'));
|
||||
|
||||
$dirmode = get_config('system','directory_mode');
|
||||
|
||||
if($d2 != intval($d1)) {
|
||||
|
||||
// If this is a directory server, request a sync with an upstream
|
||||
@ -76,7 +78,6 @@ function poller_run($argv, $argc){
|
||||
// Pull remote changes and push local changes.
|
||||
// potential issue: how do we keep from creating an endless update loop?
|
||||
|
||||
$dirmode = get_config('system','directory_mode');
|
||||
if($dirmode == DIRECTORY_MODE_SECONDARY || $dirmode == DIRECTORY_MODE_PRIMARY) {
|
||||
require_once('include/dir_fns.php');
|
||||
sync_directories($dirmode);
|
||||
@ -171,76 +172,95 @@ function poller_run($argv, $argc){
|
||||
|
||||
);
|
||||
|
||||
if(! $contacts) {
|
||||
return;
|
||||
}
|
||||
if($contacts) {
|
||||
|
||||
foreach($contacts as $contact) {
|
||||
foreach($contacts as $contact) {
|
||||
|
||||
$update = false;
|
||||
$update = false;
|
||||
|
||||
$t = $contact['abook_updated'];
|
||||
$c = $contact['abook_connected'];
|
||||
$t = $contact['abook_updated'];
|
||||
$c = $contact['abook_connected'];
|
||||
|
||||
|
||||
if($c == $t) {
|
||||
if(datetime_convert('UTC','UTC', 'now') > datetime_convert('UTC','UTC', $t . " + 1 day"))
|
||||
$update = true;
|
||||
}
|
||||
else {
|
||||
// if we've never connected with them, start the mark for death countdown from now
|
||||
|
||||
if($c == '0000-00-00 00:00:00') {
|
||||
$r = q("update abook set abook_connected = '%s' where abook_id = %d limit 1",
|
||||
dbesc(datetime_convert()),
|
||||
intval($contact['abook_id'])
|
||||
);
|
||||
$c = datetime_convert();
|
||||
$update = true;
|
||||
if($c == $t) {
|
||||
if(datetime_convert('UTC','UTC', 'now') > datetime_convert('UTC','UTC', $t . " + 1 day"))
|
||||
$update = true;
|
||||
}
|
||||
else {
|
||||
// if we've never connected with them, start the mark for death countdown from now
|
||||
|
||||
// He's dead, Jim
|
||||
if($c == '0000-00-00 00:00:00') {
|
||||
$r = q("update abook set abook_connected = '%s' where abook_id = %d limit 1",
|
||||
dbesc(datetime_convert()),
|
||||
intval($contact['abook_id'])
|
||||
);
|
||||
$c = datetime_convert();
|
||||
$update = true;
|
||||
}
|
||||
|
||||
if(strcmp(datetime_convert('UTC','UTC', 'now'),datetime_convert('UTC','UTC', $c . " + 30 day")) > 0) {
|
||||
$r = q("update abook set abook_flags = (abook_flags | %d) where abook_id = %d limit 1",
|
||||
intval(ABOOK_FLAG_ARCHIVED),
|
||||
intval($contact['abook_id'])
|
||||
);
|
||||
$update = false;
|
||||
continue;
|
||||
}
|
||||
// He's dead, Jim
|
||||
|
||||
if($contact['abook_flags'] & ABOOK_FLAG_ARCHIVED) {
|
||||
$update = false;
|
||||
continue;
|
||||
}
|
||||
if(strcmp(datetime_convert('UTC','UTC', 'now'),datetime_convert('UTC','UTC', $c . " + 30 day")) > 0) {
|
||||
$r = q("update abook set abook_flags = (abook_flags | %d) where abook_id = %d limit 1",
|
||||
intval(ABOOK_FLAG_ARCHIVED),
|
||||
intval($contact['abook_id'])
|
||||
);
|
||||
$update = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// might be dead, so maybe don't poll quite so often
|
||||
if($contact['abook_flags'] & ABOOK_FLAG_ARCHIVED) {
|
||||
$update = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// recently deceased, so keep up the regular schedule for 3 days
|
||||
// might be dead, so maybe don't poll quite so often
|
||||
|
||||
// recently deceased, so keep up the regular schedule for 3 days
|
||||
|
||||
if((strcmp(datetime_convert('UTC','UTC', 'now'),datetime_convert('UTC','UTC', $c . " + 3 day")) > 0)
|
||||
&& (strcmp(datetime_convert('UTC','UTC', 'now'),datetime_convert('UTC','UTC', $t . " + 1 day")) > 0))
|
||||
$update = true;
|
||||
if((strcmp(datetime_convert('UTC','UTC', 'now'),datetime_convert('UTC','UTC', $c . " + 3 day")) > 0)
|
||||
&& (strcmp(datetime_convert('UTC','UTC', 'now'),datetime_convert('UTC','UTC', $t . " + 1 day")) > 0))
|
||||
$update = true;
|
||||
|
||||
// After that back off and put them on a morphine drip
|
||||
|
||||
if(strcmp(datetime_convert('UTC','UTC', 'now'),datetime_convert('UTC','UTC', $t . " + 2 day")) > 0) {
|
||||
$update = true;
|
||||
}
|
||||
|
||||
// After that back off and put them on a morphine drip
|
||||
|
||||
if(strcmp(datetime_convert('UTC','UTC', 'now'),datetime_convert('UTC','UTC', $t . " + 2 day")) > 0) {
|
||||
$update = true;
|
||||
}
|
||||
|
||||
if((! $update) && (! $force))
|
||||
continue;
|
||||
|
||||
proc_run('php','include/onepoll.php',$contact['abook_id']);
|
||||
if($interval)
|
||||
@time_sleep_until(microtime(true) + (float) $interval);
|
||||
|
||||
}
|
||||
|
||||
if((! $update) && (! $force))
|
||||
continue;
|
||||
|
||||
proc_run('php','include/onepoll.php',$contact['abook_id']);
|
||||
if($interval)
|
||||
@time_sleep_until(microtime(true) + (float) $interval);
|
||||
|
||||
}
|
||||
|
||||
if($dirmode == DIRECTORY_MODE_SECONDARY || $dirmode == DIRECTORY_MODE_PRIMARY) {
|
||||
$r = q("select ud_id from updates where not ( ud_flags & %d ) and ( ud_last = '0000-00-00 00:00:00' OR ud_last > UTC_TIMESTAMP() - INTERVAL 7 DAYS) ",
|
||||
intval(UPDATE_FLAGS_UPDATED)
|
||||
);
|
||||
if($r) {
|
||||
foreach($r as $rr) {
|
||||
|
||||
// If they didn't respond when we attempted before, back off to once a day
|
||||
// After 7 days we won't bother anymore
|
||||
|
||||
if($rr['ud_last'] != '0000-00-00 00:00:00')
|
||||
if($rr['ud_last'] > datetime_convert('UTC','UTC', 'now - 1 day'))
|
||||
continue;
|
||||
proc_run('php','include/onedirsync.php',$rr['ud_id']);
|
||||
if($interval)
|
||||
@time_sleep_until(microtime(true) + (float) $interval);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@ -1532,13 +1532,21 @@ function import_directory_keywords($hash,$keywords) {
|
||||
|
||||
function update_modtime($hash,$guid,$addr,$flags = 0) {
|
||||
|
||||
q("insert into updates (ud_hash, ud_guid, ud_date, ud_flags, ud_addr ) values ( '%s', '%s', '%s', %d, '%s' )",
|
||||
dbesc($hash),
|
||||
dbesc($guid),
|
||||
dbesc(datetime_convert()),
|
||||
intval($flags),
|
||||
dbesc($addr)
|
||||
);
|
||||
if($flags) {
|
||||
q("insert into updates (ud_hash, ud_guid, ud_date, ud_flags, ud_addr ) values ( '%s', '%s', '%s', %d, '%s' )",
|
||||
dbesc($hash),
|
||||
dbesc($guid),
|
||||
dbesc(datetime_convert()),
|
||||
intval($flags),
|
||||
dbesc($addr)
|
||||
);
|
||||
}
|
||||
else {
|
||||
q("update updates set ud_flags = ( ud_flags | %d ) where ud_addr = '%s' and not (ud_flags & %d) ",
|
||||
intval(UPDATE_FLAGS_UPDATED),
|
||||
intval(UPDATE_FLAGS_UPDATED)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -823,6 +823,7 @@ CREATE TABLE IF NOT EXISTS `site` (
|
||||
`site_access` int(11) NOT NULL DEFAULT '0',
|
||||
`site_flags` int(11) NOT NULL DEFAULT '0',
|
||||
`site_update` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
|
||||
`site_sync` datetime NOT NULL,
|
||||
`site_directory` char(255) NOT NULL DEFAULT '',
|
||||
`site_register` int(11) NOT NULL DEFAULT '0',
|
||||
`site_sellpage` char(255) NOT NULL DEFAULT '',
|
||||
@ -903,6 +904,7 @@ CREATE TABLE IF NOT EXISTS `updates` (
|
||||
`ud_hash` char(128) NOT NULL,
|
||||
`ud_guid` char(255) NOT NULL DEFAULT '',
|
||||
`ud_date` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
|
||||
`ud_last` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
|
||||
`ud_flags` int(11) NOT NULL DEFAULT '0',
|
||||
`ud_addr` char(255) NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (`ud_id`),
|
||||
@ -910,7 +912,8 @@ CREATE TABLE IF NOT EXISTS `updates` (
|
||||
KEY `ud_guid` (`ud_guid`),
|
||||
KEY `ud_date` (`ud_date`),
|
||||
KEY `ud_flags` (`ud_flags`),
|
||||
KEY `ud_addr` (`ud_addr`)
|
||||
KEY `ud_addr` (`ud_addr`),
|
||||
KEY `ud_last` (`ud_last`)
|
||||
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `verify` (
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
define( 'UPDATE_VERSION' , 1074 );
|
||||
define( 'UPDATE_VERSION' , 1075 );
|
||||
|
||||
/**
|
||||
*
|
||||
@ -816,11 +816,13 @@ ADD INDEX ( `ud_addr` ) ");
|
||||
function update_r1072() {
|
||||
$r = q("ALTER TABLE `xtag` ADD `xtag_flags` INT NOT NULL DEFAULT '0',
|
||||
ADD INDEX ( `xtag_flags` ) ");
|
||||
|
||||
if($r)
|
||||
return UPDATE_SUCCESS;
|
||||
return UPDATE_FAILED;
|
||||
}
|
||||
|
||||
|
||||
function update_r1073() {
|
||||
$r1 = q("CREATE TABLE IF NOT EXISTS `source` (
|
||||
`src_id` INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY ,
|
||||
@ -836,3 +838,16 @@ function update_r1073() {
|
||||
return UPDATE_SUCCESS;
|
||||
return UPDATE_FAILED;
|
||||
}
|
||||
|
||||
function update_r1074() {
|
||||
$r1 = q("ALTER TABLE `site` ADD `site_sync` DATETIME NOT NULL AFTER `site_update` ");
|
||||
|
||||
$r2 = q("ALTER TABLE `updates` ADD `ud_last` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00' AFTER `ud_date` ,
|
||||
ADD INDEX ( `ud_last` ) ");
|
||||
|
||||
if($r1 && $r2)
|
||||
return UPDATE_SUCCESS;
|
||||
return UPDATE_FAILED;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -121,13 +121,9 @@ function dirsearch_content(&$a) {
|
||||
}
|
||||
}
|
||||
|
||||
if($mtime) {
|
||||
$qlimit = '';
|
||||
// $sql_extra .= " and xchan_hash in ( select ud_hash from updates where ud_date > '" . dbesc($mtime) . "' ) ";
|
||||
}
|
||||
|
||||
if($sort_order == 'date')
|
||||
$order = ""; // " order by ud_date desc ";
|
||||
$order = ""; // Not currently implemented
|
||||
elseif($sort_order == 'reverse')
|
||||
$order = " order by xchan_name desc ";
|
||||
else
|
||||
@ -141,11 +137,13 @@ function dirsearch_content(&$a) {
|
||||
);
|
||||
if($r) {
|
||||
foreach($r as $rr) {
|
||||
$flags = (($rr['ud_flags'] & UPDATE_FLAGS_DELETED) ? array('deleted') : array());
|
||||
$spkt['transactions'][] = array(
|
||||
'hash' => $rr['ud_hash'],
|
||||
'address' => $rr['ud_addr'],
|
||||
'transaction_id' => $rr['ud_guid'],
|
||||
'timestamp' => $rr['ud_date']
|
||||
'timestamp' => $rr['ud_date'],
|
||||
'flags' => $flags
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user