Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sync client from server using last update timestamp #846

Merged
merged 15 commits into from
Feb 27, 2017
Merged

Sync client from server using last update timestamp #846

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
7e1e575
ensure entry date can be used by the client side js code
niol Jan 15, 2017
3946683
use db lastUpdatetime to prevent uneeded computation of stats
niol Jan 15, 2017
6a2667b
get updated item status from server and show them
niol Jan 15, 2017
9c781c0
show stream more on unread view when there are new items on the server
niol Jan 15, 2017
20f740b
fix reverting ui changes after starr xhr error
niol Jan 22, 2017
e9e69f8
fix stream-more xhr sent upon each scroll event when auto_stream_more=1
niol Jan 22, 2017
3071660
improve nav-refresh button
niol Feb 12, 2017
6235fcb
use \DateTime::ATOM rather than \DateTime::ISO8601 for better compliance
niol Feb 15, 2017
31ca9e6
restore ECMAScript 5 compatibility (and IE compatibility)
niol Feb 15, 2017
1b820f7
fix typo in restore IE compatibility patch
niol Feb 15, 2017
21de209
fix typo preventing syncing starred status to the ui
niol Feb 15, 2017
a20e83b
remove js superfluous parentheses
niol Feb 27, 2017
85196bd
ensure proper types in JSON for /items/sync regardless of the db backend
niol Feb 27, 2017
7fea6c9
ensureRowTypes: prevent copying of $rows in memory for no reason
niol Feb 27, 2017
795fccc
ensureRowTypes(): do not use PHP references
niol Feb 27, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 14 additions & 17 deletions controllers/Index.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ public function home() {
$tags = $tagsDao->getWithUnread();

// load items
$itemsHtml = $this->loadItems($options, $tags);
$this->view->content = $itemsHtml;
$items = $this->loadItems($options, $tags);
$this->view->content = $items['html'];

// load stats
$itemsDao = new \daos\Items();
Expand Down Expand Up @@ -81,12 +81,14 @@ public function home() {
// ajax call = only send entries and statistics not full template
if(isset($options['ajax'])) {
$this->view->jsonSuccess(array(
"entries" => $this->view->content,
"all" => $this->view->statsAll,
"unread" => $this->view->statsUnread,
"starred" => $this->view->statsStarred,
"tags" => $this->view->tags,
"sources" => $this->view->sources
"lastUpdate" => \helpers\ViewHelper::date_iso8601($itemsDao->lastUpdate()),
"hasMore" => $items['hasMore'],
"entries" => $this->view->content,
"all" => $this->view->statsAll,
"unread" => $this->view->statsUnread,
"starred" => $this->view->statsStarred,
"tags" => $this->view->tags,
"sources" => $this->view->sources
));
}
}
Expand Down Expand Up @@ -254,15 +256,10 @@ private function loadItems($options, $tags) {
$itemsHtml .= $this->view->render('templates/item.phtml');
}

if(strlen($itemsHtml)==0) {
$itemsHtml = '<div class="stream-empty">'. \F3::get('lang_no_entries').'</div>';
} else {
if($itemDao->hasMore())
$itemsHtml .= '<div class="stream-more"><span>'. \F3::get('lang_more').'</span></div>';
$itemsHtml .= '<div class="mark-these-read"><span>'. \F3::get('lang_markread').'</span></div>';
}

return $itemsHtml;
return array(
'html' => $itemsHtml,
'hasMore' => $itemDao->hasMore()
);
}


Expand Down
44 changes: 44 additions & 0 deletions controllers/Items.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,4 +169,48 @@ public function stats() {

$this->view->jsonSuccess($stats);
}


/**
* returns updated database info (stats, item statuses)
* json
*
* @return void
*/
public function sync() {
$this->needsLoggedInOrPublicMode();

if( !array_key_exists('since', $_GET) )
$this->view->jsonError(array('sync' => 'missing since argument'));

$since = new \DateTime($_GET['since']);

$itemsDao = new \daos\Items();
$last_update = new \DateTime($itemsDao->lastUpdate());

$sync = array(
'last_update' => $last_update->format(\DateTime::ATOM),
);

if( $last_update > $since ) {
$sync['stats'] = $itemsDao->stats();

if( array_key_exists('tags', $_GET) && $_GET['tags'] == 'true' ) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now this is almost like GraphQL. Would not it be better to create a GraphQL endpoint? It could also save a lot requests during initial connection.

$tagsDao = new \daos\Tags();
$tagsController = new \controllers\Tags();
$sync['tagshtml'] = $tagsController->renderTags($tagsDao->getWithUnread());
}
if( array_key_exists('sources', $_GET) && $_GET['sources'] == 'true' ) {
$sourcesDao = new \daos\Sources();
$sourcesController = new \controllers\Sources();
$sync['sourceshtml'] = $sourcesController->renderSources($sourcesDao->getWithUnread());
}

$wantItemsStatuses = array_key_exists('items_statuses', $_GET) && $_GET['items_statuses'] == 'true';
if( $wantItemsStatuses ) {
$sync['items'] = $itemsDao->statuses($since->format(\DateTime::ATOM));
}
}
$this->view->jsonSuccess($sync);
}
}
33 changes: 33 additions & 0 deletions daos/mysql/Database.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,4 +229,37 @@ public function __construct() {
public function optimize() {
@\F3::get('db')->exec('OPTIMIZE TABLE `'.\F3::get('db_prefix').'sources`, `'.\F3::get('db_prefix').'items`');
}


/**
* Ensure row values have the appropriate PHP type. This assumes we are
* using buffered queries (sql results are in PHP memory).
*
* @param expectedRowTypes associative array mapping columns to PDO types
* @param rows array of associative array representing row results
* @return array of associative array representing row results having
* expected types
*/
public function ensureRowTypes($expectedRowTypes, &$rows) {
foreach($rows as $rowIndex => $row) {
foreach($expectedRowTypes as $column => $type) {
if( array_key_exists($column, $row) ) {
switch($type) {
case \PDO::PARAM_INT:
$value = intval($row[$column]);
break;
case \PDO::PARAM_BOOL:
if( $row[$column] == "1" )
$value = true;
else
$value = false;
break;
}
// $row is only a reference, so we change $rows[$rowIndex]
$rows[$rowIndex][$column] = $value;
}
}
}
return $rows;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do you return the value though it is mutated in-place and the returned value is never used?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought it might be more handy sometimes to directly return the call to ensureRowTypes().

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To be fair, I would prefer not using reference at all, unless the performance benefits are significant. Ideally, PHP engine would be smart enough to optimize it.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While working on this, I though about this being used on big SQL results and I red about PHP arrays passed by value being duplicated in memory by the PHP engine when modified within the function. I wanted to prevent this. It is trivial to remove though.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For now, I would remove the returns, if we need them in the future, they can always be re-added. Though, in the long term I would like to drop this altogether and use some abstraction like https://nextras.org/orm/

}
}
36 changes: 36 additions & 0 deletions daos/mysql/Items.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,42 @@ public function stats() {
'.$this->stmt->sumBool('unread').' AS unread,
'.$this->stmt->sumBool('starred').' AS starred
FROM '.\F3::get('db_prefix').'items;');
$this->ensureRowTypes(array('total' => \PDO::PARAM_INT,
'unread' => \PDO::PARAM_INT,
'starred' => \PDO::PARAM_INT),
$res);
return $res[0];
}


/**
* returns the datetime of the last item update or user action in db
*
* @return timestamp
*/
public function lastUpdate() {
$res = \F3::get('db')->exec('SELECT
MAX(updatetime) AS last_update_time
FROM '.\F3::get('db_prefix').'items;');
return $res[0]['last_update_time'];
}


/**
* returns the statuses of items last update
*
* @param date since to return item statuses
* @return array of unread, starred, etc. status of specified items
*/
public function statuses($since) {
$res = \F3::get('db')->exec('SELECT id, unread, starred
Copy link
Member

@jtojnar jtojnar Feb 17, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it possible to use proper booleans? Otherwise the sync will mark all changed items as starred and unread since unlike in PHP !!"0" === true in JS:

$ php -r 'var_dump(!!"0");'
Command line code:1:
bool(false)

$ node -e 'console.log(!!"0")'
true
  1. Open stream
  2. In another tab mark some item from the stream as read
  3. Wait for synchronization, the item will remain unread in the first tab

FROM '.\F3::get('db_prefix').'items
WHERE '.\F3::get('db_prefix').'items.updatetime > :since;',
array(':since' => array($since, \PDO::PARAM_STR)));
$this->ensureRowTypes(array('id' => \PDO::PARAM_INT,
'unread' => \PDO::PARAM_BOOL,
'starred' => \PDO::PARAM_BOOL),
$res);
return $res;
}
}
21 changes: 21 additions & 0 deletions daos/pgsql/Database.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,13 @@ public function __construct() {
'INSERT INTO version (version) VALUES (9);'
));
}
if(strnatcmp($version, "10") < 0) {
\F3::get('db')->exec(array(
'ALTER TABLE items ALTER COLUMN datetime SET DATA TYPE timestamp(0) with time zone;',
'ALTER TABLE items ALTER COLUMN updatetime SET DATA TYPE timestamp(0) with time zone;',
'INSERT INTO version (version) VALUES (10);'
));
}
}

// just initialize once
Expand All @@ -227,4 +234,18 @@ public function __construct() {
public function optimize() {
\F3::get('db')->exec("VACUUM ANALYZE");
}


/**
* Ensure row values have the appropriate PHP type. This assumes we are
* using buffered queries (sql results are in PHP memory).
*
* @param expectedRowTypes associative array mapping columns to PDO types
* @param rows array of associative array representing row results
* @return array of associative array representing row results having
* expected types
*/
public function ensureRowTypes($expectedRowTypes, $rows) {
return $rows; // pgsql returns correct PHP types
}
}
13 changes: 13 additions & 0 deletions helpers/ViewHelper.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,19 @@ public function dateago($datestr) {
return \F3::get('lang_timestamp', $date->getTimestamp());
}


/**
* Return ISO8601 formatted date
*
* @param sql date
* @return string
*/
public static function date_iso8601($datestr) {
$date = new \DateTime($datestr);
return $date->format(\DateTime::ATOM);
}


/**
* Proxify imgs through atmos/camo when not https
*
Expand Down
2 changes: 2 additions & 0 deletions index.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
'public/js/selfoss-base.js',
'public/js/selfoss-shares.js',
'public/js/selfoss-db.js',
'public/js/selfoss-ui.js',
'public/js/selfoss-events.js',
'public/js/selfoss-events-navigation.js',
'public/js/selfoss-events-search.js',
Expand Down Expand Up @@ -68,6 +69,7 @@
$f3->route('GET /tags', 'controllers\Tags->listTags'); // json
$f3->route('GET /tagslist', 'controllers\Tags->tagslist'); // html
$f3->route('GET /stats', 'controllers\Items->stats'); // json
$f3->route('GET /items/sync', 'controllers\Items->sync'); // json
$f3->route('GET /sources/stats', 'controllers\Sources->stats'); // json

// only loggedin users
Expand Down
25 changes: 21 additions & 4 deletions public/css/style.css
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,10 @@ body * {
#nav-refresh {
background:url(images/nav-refresh.png) no-repeat center center #272325;
}

#nav-refresh.loading {
background:url(images/ajax-loader.gif) center center no-repeat;
}

#nav-settings {
background:url(images/nav-sources.png) no-repeat center center #272325;
Expand Down Expand Up @@ -398,14 +402,18 @@ body * {

/* content */

#content {
#content, #stream-buttons {
margin-left:220px;
}

#content {
padding-top:20px;
padding-bottom:20px;
}

#content .stream-empty {
.stream-empty {
text-align:center;
display: none;
}

#content.loading {
Expand Down Expand Up @@ -686,6 +694,7 @@ body * {
font-weight:bold;
cursor:pointer;
text-align:center;
display: none;
}

.touch .stream-more {
Expand Down Expand Up @@ -1244,11 +1253,15 @@ body.publicmode.notloggedin .entry-unread {
margin-left:10px;
}

#content {
#content, #stream-buttons {
margin:0;
padding:0;
width:100%;
}

.stream-empty {
padding-top: 20px;
}

.source,
.entry {
Expand Down Expand Up @@ -1390,12 +1403,16 @@ body.publicmode.notloggedin .entry-unread {
width:135px;
}

#content {
#content, #stream-buttons {
margin-left:165px;
margin-top:0;
margin-right:0;
padding:0;
}

.stream-empty {
padding-top: 20px;
}

}

Expand Down
Loading