dlhist: rename to proc-cache.
A new datastructure is about to take dlhist place. dlhist is currently implemented as a mixture of an "process cache" that should record what rss items has been processed (that is why the url is used as a unique identifier), but right now it only stores an url if it has been downloaded. A new datastructure that should be "download history" shall be implemented, that will keep track of what title and where it has been downloaded to. this will make it possible to only download an rss title to a location once. Splitting this datastructure into two separated structures is trivial as a "process cache" will threat URL's as a unique identifier and a "download history" will threat the title in an rss item as a unique identifier (and also track it's destinations). This commit does not change any functionality, I just rename this to keep the "dlhist" prefix and source files clear for when implementing the real dlhist.
This commit is contained in:
parent
90365e9de2
commit
1350330dd2
4 changed files with 34 additions and 33 deletions
3
Makefile
3
Makefile
|
|
@ -10,7 +10,8 @@ all : $(PROGRAMS)
|
|||
install : $(PROGRAMS)
|
||||
cp $^ $(HOME)/bin/
|
||||
|
||||
dlight : dlight.o buffer.o env.o http.o rss.o lockfile.o filter.o cconf.o dlhist.o error.o
|
||||
dlight : dlight.o buffer.o env.o http.o rss.o lockfile.o filter.o cconf.o \
|
||||
proc-cache.o error.o
|
||||
dlight-compile : compile.o buffer.o env.o lockfile.o filter.o cconf.o error.o
|
||||
dlight-read-config : read-config.o buffer.o env.o cconf.o error.o
|
||||
dlight-filter-check: filter-check.o filter.o error.o
|
||||
|
|
|
|||
14
dlight.c
14
dlight.c
|
|
@ -9,12 +9,12 @@
|
|||
#include "env.h"
|
||||
#include "error.h"
|
||||
#include "cconf.h"
|
||||
#include "dlhist.h"
|
||||
#include "proc-cache.h"
|
||||
#include "filter.h"
|
||||
#include "http.h"
|
||||
#include "rss.h"
|
||||
|
||||
#define DLHIST_PURGE_INTERVAL (60*60*6) /* 6 hours (in seconds) */
|
||||
#define PROC_CACHE_PURGE_INTERVAL (60*60*6) /* 6 hours (in seconds) */
|
||||
|
||||
static int write_http_file(struct http_file *file, const char *dest) {
|
||||
|
||||
|
|
@ -45,7 +45,7 @@ static void process_items(rss_t rss, struct target *t) {
|
|||
|
||||
struct http_file *file = NULL;
|
||||
|
||||
if (dlhist_lookup(item.link))
|
||||
if (proc_cache_lookup(item.link))
|
||||
continue;
|
||||
|
||||
for(i=0; i < t->nr; i++) {
|
||||
|
|
@ -70,7 +70,7 @@ static void process_items(rss_t rss, struct target *t) {
|
|||
printf("Downloaded: %s (%s) to %s\n",
|
||||
item.title, item.link, filter->dest);
|
||||
|
||||
dlhist_update(item.link);
|
||||
proc_cache_update(item.link);
|
||||
}
|
||||
|
||||
http_free_file(file);
|
||||
|
|
@ -82,7 +82,7 @@ static void process(struct cconf *config) {
|
|||
int i;
|
||||
struct buffer *data;
|
||||
|
||||
dlhist_purge(DLHIST_PURGE_INTERVAL);
|
||||
proc_cache_purge(PROC_CACHE_PURGE_INTERVAL);
|
||||
|
||||
for(i=0; i < config->nr; i++) {
|
||||
struct target *t = config->target + i;
|
||||
|
|
@ -118,12 +118,12 @@ int main(int argc, char *argv[]) {
|
|||
return 1;
|
||||
}
|
||||
|
||||
if (dlhist_open() < 0)
|
||||
if (proc_cache_open() < 0)
|
||||
return 1;
|
||||
|
||||
process(config);
|
||||
|
||||
dlhist_close();
|
||||
proc_cache_close();
|
||||
cconf_free(config);
|
||||
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* dlhist.c
|
||||
/* proc-cache.c
|
||||
*
|
||||
* Copyright (C) 2011 Henrik Hautakoski <henrik@fiktivkod.org>
|
||||
*
|
||||
|
|
@ -29,11 +29,11 @@
|
|||
#include <time.h>
|
||||
#include "env.h"
|
||||
#include "lockfile.h"
|
||||
#include "dlhist.h"
|
||||
#include "proc-cache.h"
|
||||
|
||||
/* \195 D L H */
|
||||
#define SIGNATURE 0xC3444C48
|
||||
#define STORAGE_FILE "dlhist"
|
||||
/* \175 D P C */
|
||||
#define SIGNATURE 0xAF445043
|
||||
#define STORAGE_FILE "proc-cache"
|
||||
|
||||
#define TABLE_MIN_SIZE 128
|
||||
|
||||
|
|
@ -191,7 +191,7 @@ static void build_table(const char *buf, size_t entries) {
|
|||
}
|
||||
}
|
||||
|
||||
int dlhist_open() {
|
||||
int proc_cache_open() {
|
||||
|
||||
char filename[4096], *buf = NULL;
|
||||
int ret = -1, fd = -1;
|
||||
|
|
@ -209,7 +209,7 @@ int dlhist_open() {
|
|||
|
||||
fd = open(filename, O_CREAT | O_RDONLY, 0600);
|
||||
if (fd < 0 || fstat(fd, &st) < 0) {
|
||||
perror("dlhist_open");
|
||||
perror("proc_cache_open");
|
||||
goto error;
|
||||
}
|
||||
|
||||
|
|
@ -225,7 +225,7 @@ int dlhist_open() {
|
|||
hdr = (struct header *) buf;
|
||||
if (hdr->signature != htonl(SIGNATURE) ||
|
||||
hdr->version != htonl(1)) {
|
||||
fprintf(stderr, "dlhist_open: Invalid header\n");
|
||||
fprintf(stderr, "proc_cache_open: Invalid header\n");
|
||||
goto error;
|
||||
}
|
||||
|
||||
|
|
@ -236,7 +236,7 @@ int dlhist_open() {
|
|||
|
||||
if (entries * HE_SZ > st.st_size - offset) {
|
||||
fprintf(stderr,
|
||||
"dlhist_open: file truncated. "
|
||||
"proc_cache_open: file truncated. "
|
||||
"expected atleast '%lu' bytes, got '%lu'\n",
|
||||
entries * HE_SZ, st.st_size - offset);
|
||||
goto error;
|
||||
|
|
@ -255,7 +255,7 @@ error:
|
|||
return ret;
|
||||
}
|
||||
|
||||
int dlhist_lookup(const char *url) {
|
||||
int proc_cache_lookup(const char *url) {
|
||||
|
||||
if (table_size) {
|
||||
struct hash_entry *he = lookup(url);
|
||||
|
|
@ -264,7 +264,7 @@ int dlhist_lookup(const char *url) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
void dlhist_update(const char *url) {
|
||||
void proc_cache_update(const char *url) {
|
||||
|
||||
struct hash_entry *he;
|
||||
|
||||
|
|
@ -283,7 +283,7 @@ void dlhist_update(const char *url) {
|
|||
}
|
||||
}
|
||||
|
||||
void dlhist_purge(unsigned int timestamp) {
|
||||
void proc_cache_purge(unsigned int timestamp) {
|
||||
|
||||
unsigned int i, t = 0, now = time(NULL);
|
||||
|
||||
|
|
@ -301,7 +301,7 @@ void dlhist_purge(unsigned int timestamp) {
|
|||
resize_table();
|
||||
}
|
||||
|
||||
void dlhist_flush() {
|
||||
void proc_cache_flush() {
|
||||
|
||||
int i;
|
||||
struct header hdr;
|
||||
|
|
@ -339,9 +339,9 @@ void dlhist_flush() {
|
|||
commit_lock(&lock);
|
||||
}
|
||||
|
||||
void dlhist_close() {
|
||||
void proc_cache_close() {
|
||||
|
||||
dlhist_flush();
|
||||
proc_cache_flush();
|
||||
|
||||
release_lock(&lock);
|
||||
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
/* dlhist.h
|
||||
/* proc-cache.h
|
||||
*
|
||||
* Copyright (C) 2011 Henrik Hautakoski <henrik@fiktivkod.org>
|
||||
*
|
||||
|
|
@ -17,19 +17,19 @@
|
|||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
#ifndef DLHIST_H
|
||||
#define DLHIST_H
|
||||
#ifndef PROC_CACHE_H
|
||||
#define PROC_CACHE_H
|
||||
|
||||
int dlhist_open();
|
||||
int proc_cache_open();
|
||||
|
||||
int dlhist_lookup(const char *url);
|
||||
int proc_cache_lookup(const char *url);
|
||||
|
||||
void dlhist_update(const char *url);
|
||||
void proc_cache_update(const char *url);
|
||||
|
||||
void dlhist_purge(unsigned int timestamp);
|
||||
void proc_cache_purge(unsigned int timestamp);
|
||||
|
||||
void dlhist_flush();
|
||||
void proc_cache_flush();
|
||||
|
||||
void dlhist_close();
|
||||
void proc_cache_close();
|
||||
|
||||
#endif /* DLHIST */
|
||||
#endif /* PROC_CACHE_H */
|
||||
Reference in a new issue