Skip to content

Commit

Permalink
updated LinkTable invalidation
Browse files Browse the repository at this point in the history
  • Loading branch information
fangfufu committed May 11, 2024
1 parent a5a5344 commit 0747566
Show file tree
Hide file tree
Showing 2 changed files with 52 additions and 57 deletions.
12 changes: 12 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Fixed
- The refreshed LinkTable is now saved
(https://github.com/fangfufu/httpdirfs/issues/141).
- Only one LinkTable of the same directory is created when the cache mode is
enabled (https://github.com/fangfufu/httpdirfs/issues/140).
- Cache mode noe works correctly witht escaped URL
(https://github.com/fangfufu/httpdirfs/issues/138).

## Changed
- Improved LinkTable caching. LinkTable invalidation is now purely based on
timeout.

## [1.2.5] - 2023-02-24

### Fixed
Expand Down
97 changes: 40 additions & 57 deletions src/link.c
Original file line number Diff line number Diff line change
Expand Up @@ -571,31 +571,14 @@ LinkTable *LinkTable_alloc(const char *url)

LinkTable *LinkTable_new(const char *url)
{
LinkTable *linktbl = LinkTable_alloc(url);
linktbl->index_time = time(NULL);
lprintf(debug, "linktbl->index_time: %d\n", linktbl->index_time);

/*
* start downloading the base URL
*/
TransferStruct ts = Link_download_full(linktbl->links[0]);
if (ts.curr_size == 0) {
LinkTable_free(linktbl);
return NULL;
}

/*
* Otherwise parsed the received data
*/
GumboOutput *output = gumbo_parse(ts.data);
HTML_to_LinkTable(url, output->root, linktbl);
gumbo_destroy_output(&kGumboDefaultOptions, output);
FREE(ts.data);

int skip_fill = 0;
char *unescaped_path;
unescaped_path =
curl_easy_unescape(NULL, url + ROOT_LINK_OFFSET, 0, NULL);
LinkTable *linktbl = NULL;

/*
* Attempt to load the LinkTable from the disk.
*/
if (CACHE_SYSTEM_INIT) {
CacheDir_create(unescaped_path);
LinkTable *disk_linktbl;
Expand All @@ -613,57 +596,56 @@ LinkTable *LinkTable_new(const char *url)
time_now - disk_linktbl->index_time,
CONFIG.refresh_timeout);
LinkTable_free(disk_linktbl);
}
/*
* Check if there are inconsistencies for the on-disk LinkTable
*/

if (disk_linktbl->num == linktbl->num) {
LinkTable_free(linktbl);
linktbl = disk_linktbl;
skip_fill = 1;
} else {
lprintf(info,
"disk_linktbl->num: %d, linktbl->num: %d\n",
disk_linktbl->num, linktbl->num);
lprintf(info, "Refreshing LinkTable for %s\n",
url + ROOT_LINK_OFFSET);
LinkTable_free(disk_linktbl);
linktbl = disk_linktbl;
}
}
}

if (!skip_fill) {
/*
* Download a new LinkTable because we didn't manange to load it from the
* disk
*/
if (!linktbl) {
linktbl->index_time = time(NULL);
lprintf(debug, "linktbl->index_time: %d\n", linktbl->index_time);

/*
* Fill in the link table
*/
LinkTable_fill(linktbl);
} else {
* start downloading the base URL
*/
TransferStruct ts = Link_download_full(linktbl->links[0]);
if (ts.curr_size == 0) {
LinkTable_free(linktbl);
return NULL;
}

/*
* Fill in the holes in the link table
*/
LinkTable_invalid_reset(linktbl);
LinkTable_uninitialised_fill(linktbl);
}
* Otherwise parsed the received data
*/
GumboOutput *output = gumbo_parse(ts.data);
HTML_to_LinkTable(url, output->root, linktbl);
gumbo_destroy_output(&kGumboDefaultOptions, output);
FREE(ts.data);

/*
* Save the link table
*/
if (CACHE_SYSTEM_INIT) {
if (LinkTable_disk_save(linktbl, unescaped_path)) {
lprintf(error, "Failed to save the LinkTable!\n");
LinkTable_fill(linktbl);

/*
* Save the link table
*/
if (CACHE_SYSTEM_INIT) {
if (LinkTable_disk_save(linktbl, unescaped_path)) {
lprintf(error, "Failed to save the LinkTable!\n");
}
}
}
free(unescaped_path);

#ifdef DEBUG
static int i = 0;
lprintf(debug, "!!!!Calling LinkTable_new for the %d time!!!!\n", i);
i++;
#endif

free(unescaped_path);
LinkTable_print(linktbl);

return linktbl;
}

Expand Down Expand Up @@ -791,8 +773,9 @@ LinkTable *LinkTable_disk_open(const char *dirn)
lprintf(error,
"cannot close the file pointer, %s\n", strerror(errno));
}
return linktbl;

FREE(path);
return linktbl;
}

LinkTable *path_to_Link_LinkTable_new(const char *path)
Expand Down

0 comments on commit 0747566

Please sign in to comment.