archive/dlight
Archived
1
0
Fork 0
Code Activity

dlight.c: only download a file once.

Browse source 
When going through the filter list for an item, we download and store the item
everytime a filter is matched.
This patch allowes an item to be downloaded the first time a filter
match and save the data throughout the rest of the list, so all
other matches never downloads the item again but uses the data in memory.
This commit is contained in:
Henrik Hautakoski 2011-09-29 21:34:34 +02:00
parent eb0702cd93
commit 02814cc1f7
1 changed files with 40 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

44
dlight.c
View file

@ -1,6 +1,10 @@
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h> #include <errno.h>
#include "env.h" #include "env.h"
#include "error.h" #include "error.h"
@ -12,6 +16,26 @@
#define DLHIST_PURGE_INTERVAL (60*60*6) /* 6 hours (in seconds) */ #define DLHIST_PURGE_INTERVAL (60*60*6) /* 6 hours (in seconds) */
static int write_http_file(struct http_file *file, const char *dest) {
char path[4096];
int rc, fd;
snprintf(path, sizeof(path), "%s/%s",
dest, file->filename);
fd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0664);
if (fd < 0 && errno != EEXIST) {
error("failed to write file: %s\n", path);
return -1;
}
rc = write(fd, file->data.block, file->data.len);
close(fd);
return rc;
}
static void process_items(rss_t rss, struct target *t) { static void process_items(rss_t rss, struct target *t) {
int i; int i;
@ -19,6 +43,8 @@ static void process_items(rss_t rss, struct target *t) {
while(rss_walk_next(rss, &item)) { while(rss_walk_next(rss, &item)) {
struct http_file *file = NULL;
if (dlhist_lookup(item.link)) if (dlhist_lookup(item.link))
continue; continue;
@ -28,17 +54,27 @@ static void process_items(rss_t rss, struct target *t) {
if (!filter_match(filter->pattern, item.title)) if (!filter_match(filter->pattern, item.title))
continue; continue;
if (http_download_file(item.link, filter->dest) < 0 && /* fetch the file if we haven't already. */
errno != EEXIST) { if (file == NULL) {
error("download failed: %s", strerror(errno)); file = http_fetch_file(item.link);
continue; if (file == NULL) {
error("download failed: %s",
strerror(errno));
continue;
}
} }
/* save file to disk. */
if (write_http_file(file, filter->dest) < 0)
continue;
printf("Downloaded: %s (%s) to %s\n", printf("Downloaded: %s (%s) to %s\n",
item.title, item.link, filter->dest); item.title, item.link, filter->dest);
dlhist_update(item.link); dlhist_update(item.link);
} }
http_free_file(file);
} }
} }