From 5969698ff6028947e097927e2391406481a36290 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Orsini Date: Thu, 10 May 2012 09:03:15 +0000 Subject: [PATCH] use file cache for distros, refactoring http fcts --- po/Makefile | 2 +- po/Makefile.in | 2 +- src/Makefile.am | 1 + src/Makefile.in | 6 ++- src/fcache.c | 11 ++-- src/http.c | 162 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/http.h | 27 ++++++++++ src/lp_ws.c | 137 ++++++----------------------------------------- src/main.c | 3 +- 9 files changed, 221 insertions(+), 130 deletions(-) create mode 100644 src/http.c create mode 100644 src/http.h diff --git a/po/Makefile b/po/Makefile index 69bf449..b5acc35 100644 --- a/po/Makefile +++ b/po/Makefile @@ -22,7 +22,7 @@ srcdir = . top_srcdir = .. -prefix = /usr +prefix = /tmp/p exec_prefix = ${prefix} datarootdir = ${prefix}/share datadir = ${datarootdir} diff --git a/po/Makefile.in b/po/Makefile.in index cbbdd55..9fdca0c 100644 --- a/po/Makefile.in +++ b/po/Makefile.in @@ -22,7 +22,7 @@ srcdir = . top_srcdir = .. -prefix = /usr +prefix = /tmp/p exec_prefix = ${prefix} datarootdir = ${prefix}/share datadir = ${datarootdir} diff --git a/src/Makefile.am b/src/Makefile.am index 5add147..dc6e709 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -12,6 +12,7 @@ ppastats_SOURCES = \ cache.h cache.c\ fcache.h fcache.c\ html.h html.c\ + http.h http.c\ io.h io.c\ list.h list.c\ log.h log.c\ diff --git a/src/Makefile.in b/src/Makefile.in index 20ce8c4..217d5bf 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -54,8 +54,8 @@ CONFIG_CLEAN_VPATH_FILES = am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(man1dir)" PROGRAMS = $(bin_PROGRAMS) am_ppastats_OBJECTS = cache.$(OBJEXT) fcache.$(OBJEXT) html.$(OBJEXT) \ - io.$(OBJEXT) list.$(OBJEXT) log.$(OBJEXT) lp.$(OBJEXT) \ - lp_json.$(OBJEXT) lp_ws.$(OBJEXT) main.$(OBJEXT) \ + http.$(OBJEXT) io.$(OBJEXT) list.$(OBJEXT) log.$(OBJEXT) \ + lp.$(OBJEXT) lp_json.$(OBJEXT) lp_ws.$(OBJEXT) main.$(OBJEXT) \ ppastats.$(OBJEXT) str.$(OBJEXT) ppastats_OBJECTS = $(am_ppastats_OBJECTS) ppastats_LDADD = $(LDADD) @@ -231,6 +231,7 @@ ppastats_SOURCES = \ cache.h cache.c\ fcache.h fcache.c\ html.h html.c\ + http.h http.c\ io.h io.c\ list.h list.c\ log.h log.c\ @@ -326,6 +327,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cache.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fcache.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/html.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/io.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/list.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/log.Po@am__quote@ diff --git a/src/fcache.c b/src/fcache.c index 89c66d4..1a4623c 100644 --- a/src/fcache.c +++ b/src/fcache.c @@ -59,12 +59,15 @@ static char *key_to_path(const char *key) char *path; const char *dir; + if (!key || !*key || *key != '/') + return NULL; + dir = get_cache_dir(); if (!dir) return NULL; - path = malloc(strlen(dir) + 1 + strlen(key) + strlen("/data.json") + 1); - sprintf(path, "%s%s/data.json", dir, key); + path = malloc(strlen(dir) + 1 + strlen(key) + strlen("/data") + 1); + sprintf(path, "%s%s/data", dir, key); return path; } @@ -74,8 +77,10 @@ char *fcache_get(const char *key) char *path, *content; path = key_to_path(key); - if (!path) + if (!path) { + log_err(_("file cache, invalid key: %s"), key); return NULL; + } content = file_get_content(path); diff --git a/src/http.c b/src/http.c new file mode 100644 index 0000000..f2351c9 --- /dev/null +++ b/src/http.c @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2011-2012 jeanfi@gmail.com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA + */ + +#include +#define _(String) gettext(String) + +#include +#include +#include + +#include + +#include "fcache.h" +#include "http.h" +#include "log.h" + +static const int DEFAULT_FETCH_RETRIES = 10; + +static CURL *curl; + +struct ucontent { + char *data; + size_t len; +}; + +static size_t cbk_curl(void *buffer, size_t size, size_t nmemb, void *userp) +{ + size_t realsize = size * nmemb; + struct ucontent *mem = (struct ucontent *)userp; + + mem->data = realloc(mem->data, mem->len + realsize + 1); + + memcpy(&(mem->data[mem->len]), buffer, realsize); + mem->len += realsize; + mem->data[mem->len] = 0; + + return realsize; +} + +static void init() +{ + if (!curl) { + log_debug(_("initializing CURL")); + curl_global_init(CURL_GLOBAL_ALL); + curl = curl_easy_init(); + } + + if (!curl) + exit(EXIT_FAILURE); +} + +static char *fetch_url(const char *url) +{ + struct ucontent *content = malloc(sizeof(struct ucontent)); + char *result; + long code; + int retries; + unsigned int s; + + log_debug(_("fetch_url(): %s"), url); + + init(); + + result = NULL; + + retries = DEFAULT_FETCH_RETRIES; + + retrieve: + content->data = malloc(1); + content->data[0] = '\0'; + content->len = 0; + + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_VERBOSE, 0); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cbk_curl); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, content); + curl_easy_setopt(curl, CURLOPT_USERAGENT, "ppastats/0.0"); + + if (curl_easy_perform(curl) == CURLE_OK) { + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); + + switch (code) { + case 200: + result = content->data; + break; + case 500: + case 502: + case 503: + case 504: + log_err(_("Fetch failed with code %ld for URL %s"), + code, + url); + + if (retries) { + s = 2 * (DEFAULT_FETCH_RETRIES - retries) + 2; + log_debug(_("Wait %ds before retry"), s); + sleep(s); + + free(content->data); + retries--; + goto retrieve; + } + + break; + default: + log_err(_("Fetch failed with code %ld for URL %s"), + code, + url); + } + } + + if (!result) + free(content->data); + + free(content); + + return result; +} + +void http_cleanup() +{ + log_debug(_("cleanup CURL")); + + curl_easy_cleanup(curl); + curl_global_cleanup(); +} + +char *get_url_content(const char *url, unsigned int use_cache) +{ + char *content; + + content = NULL; + + if (use_cache) + content = fcache_get(url + 7); + + if (!content) { + content = fetch_url(url); + } + + if (use_cache && content) + fcache_put(url + 7, content); + + return content; +} + diff --git a/src/http.h b/src/http.h new file mode 100644 index 0000000..c85b004 --- /dev/null +++ b/src/http.h @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2011-2012 jeanfi@gmail.com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA + */ + +#ifndef _PPASTATS_HTTP_H_ +#define _PPASTATS_HTTP_H_ + +char *get_url_content(const char *url, unsigned int use_cache); + +void http_cleanup(); + +#endif diff --git a/src/lp_ws.c b/src/lp_ws.c index 023ba9b..63a8d36 100644 --- a/src/lp_ws.c +++ b/src/lp_ws.c @@ -22,13 +22,12 @@ #include #include -#include -#include #include #include "cache.h" #include "fcache.h" +#include "http.h" #include "list.h" #include "log.h" #include "lp_ws.h" @@ -41,115 +40,12 @@ static const char *QUERY_GET_DOWNLOAD_COUNT = "?ws.op=getDownloadCount"; static const char * QUERY_GET_DAILY_DOWNLOAD_TOTALS = "?ws.op=getDailyDownloadTotals"; -static const int DEFAULT_FETCH_RETRIES = 10; - -static CURL *curl; - -struct ucontent { - char *data; - size_t len; -}; - -static size_t cbk_curl(void *buffer, size_t size, size_t nmemb, void *userp) -{ - size_t realsize = size * nmemb; - struct ucontent *mem = (struct ucontent *)userp; - - mem->data = realloc(mem->data, mem->len + realsize + 1); - - memcpy(&(mem->data[mem->len]), buffer, realsize); - mem->len += realsize; - mem->data[mem->len] = 0; - - return realsize; -} - -static void init() -{ - if (!curl) { - log_debug(_("initializing CURL")); - curl_global_init(CURL_GLOBAL_ALL); - curl = curl_easy_init(); - } - - if (!curl) - exit(EXIT_FAILURE); -} - -static char *fetch_url(const char *url) -{ - struct ucontent *content = malloc(sizeof(struct ucontent)); - char *result; - long code; - int retries; - unsigned int s; - - log_debug(_("fetch_url(): %s"), url); - - init(); - - result = NULL; - - retries = DEFAULT_FETCH_RETRIES; - - retrieve: - content->data = malloc(1); - content->data[0] = '\0'; - content->len = 0; - - curl_easy_setopt(curl, CURLOPT_URL, url); - curl_easy_setopt(curl, CURLOPT_VERBOSE, 0); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cbk_curl); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, content); - curl_easy_setopt(curl, CURLOPT_USERAGENT, "ppastats/0.0"); - - if (curl_easy_perform(curl) == CURLE_OK) { - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); - - switch (code) { - case 200: - result = content->data; - break; - case 500: - case 502: - case 503: - case 504: - log_err(_("Fetch failed with code %ld for URL %s"), - code, - url); - - if (retries) { - s = 2 * (DEFAULT_FETCH_RETRIES - retries) + 2; - log_debug(_("Wait %ds before retry"), s); - sleep(s); - - free(content->data); - retries--; - goto retrieve; - } - - break; - default: - log_err(_("Fetch failed with code %ld for URL %s"), - code, - url); - } - } - - if (!result) - free(content->data); - - free(content); - - return result; -} - static json_object *get_json_object(const char *url) { json_object *obj = NULL; char *body; - body = fetch_url(url); + body = get_url_content(url, 0); if (body) { obj = json_tokener_parse(body); @@ -242,14 +138,10 @@ const struct distro_arch_series *get_distro_arch_series(const char *url) if (distro) return (struct distro_arch_series *)distro; - content = fcache_get(url + 7); - if (!content) { - content = fetch_url(url); - if (content) - fcache_put(url + 7, content); - else - return NULL; - } + content = get_url_content(url, 1); + + if (!content) + return NULL; obj = json_tokener_parse(content); @@ -271,12 +163,20 @@ const struct distro_series *get_distro_series(const char *url) { json_object *obj; const struct distro_series *distro; + char *content; distro = cache_get(url); if (distro) return (struct distro_series *)distro; - obj = get_json_object(url); + content = get_url_content(url, 1); + + if (!content) + return NULL; + + obj = json_tokener_parse(content); + + free(content); if (!obj) return NULL; @@ -314,10 +214,3 @@ struct daily_download_total **get_daily_download_totals(const char *binary_url) return result; } -void lp_ws_cleanup() -{ - log_debug(_("cleanup CURL")); - - curl_easy_cleanup(curl); - curl_global_cleanup(); -} diff --git a/src/main.c b/src/main.c index ef215eb..7bd87fc 100644 --- a/src/main.c +++ b/src/main.c @@ -27,6 +27,7 @@ #include "cache.h" #include "html.h" +#include "http.h" #include "log.h" #include "lp_ws.h" #include "config.h" @@ -204,7 +205,7 @@ int main(int argc, char **argv) /* for valgrind.... */ free(package_status); free(output_dir); - lp_ws_cleanup(); + http_cleanup(); cache_cleanup(); exit(EXIT_SUCCESS); -- 2.7.4