X-Git-Url: http://git.wpitchoune.net/gitweb/?a=blobdiff_plain;f=src%2Flp_ws.c;h=023ba9b28f17e58af7f096adf0bea4fac4edbaad;hb=be68914bbcbfde6dbc51e20075691afbab3c2489;hp=2370d36664035f2c8628e5b80da1b41ff8255ab1;hpb=9412dc53bab346c119e4d1502e65cd7c68487d30;p=ppastats.git diff --git a/src/lp_ws.c b/src/lp_ws.c index 2370d36..023ba9b 100644 --- a/src/lp_ws.c +++ b/src/lp_ws.c @@ -1,39 +1,47 @@ /* - Copyright (C) 2011 jeanfi@gmail.com - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - 02110-1301 USA -*/ + * Copyright (C) 2011-2012 jeanfi@gmail.com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA + */ + +#include +#define _(String) gettext(String) #include #include +#include #include #include #include "cache.h" +#include "fcache.h" #include "list.h" +#include "log.h" #include "lp_ws.h" #include "lp_json.h" #include "ppastats.h" -#define QUERY_GET_PUBLISHED_BINARIES \ - "?ws.op=getPublishedBinaries" -#define QUERY_GET_DOWNLOAD_COUNT "?ws.op=getDownloadCount" -#define QUERY_GET_DAILY_DOWNLOAD_TOTALS \ - "?ws.op=getDailyDownloadTotals" +static const char * +QUERY_GET_PUBLISHED_BINARIES = "?ws.op=getPublishedBinaries&ws.size=300"; +static const char *QUERY_GET_DOWNLOAD_COUNT = "?ws.op=getDownloadCount"; +static const char * +QUERY_GET_DAILY_DOWNLOAD_TOTALS = "?ws.op=getDailyDownloadTotals"; + +static const int DEFAULT_FETCH_RETRIES = 10; static CURL *curl; @@ -56,23 +64,35 @@ static size_t cbk_curl(void *buffer, size_t size, size_t nmemb, void *userp) return realsize; } -static char *fetch_url(const char *url) +static void init() { - struct ucontent *content = malloc(sizeof(struct ucontent)); - char *result = NULL; - long code; - - if (debug) - printf("DEBUG: fetch_url %s\n", url); - if (!curl) { + log_debug(_("initializing CURL")); curl_global_init(CURL_GLOBAL_ALL); curl = curl_easy_init(); } if (!curl) exit(EXIT_FAILURE); +} +static char *fetch_url(const char *url) +{ + struct ucontent *content = malloc(sizeof(struct ucontent)); + char *result; + long code; + int retries; + unsigned int s; + + log_debug(_("fetch_url(): %s"), url); + + init(); + + result = NULL; + + retries = DEFAULT_FETCH_RETRIES; + + retrieve: content->data = malloc(1); content->data[0] = '\0'; content->len = 0; @@ -84,10 +104,36 @@ static char *fetch_url(const char *url) curl_easy_setopt(curl, CURLOPT_USERAGENT, "ppastats/0.0"); if (curl_easy_perform(curl) == CURLE_OK) { - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); - if (code == 200) + + switch (code) { + case 200: result = content->data; + break; + case 500: + case 502: + case 503: + case 504: + log_err(_("Fetch failed with code %ld for URL %s"), + code, + url); + + if (retries) { + s = 2 * (DEFAULT_FETCH_RETRIES - retries) + 2; + log_debug(_("Wait %ds before retry"), s); + sleep(s); + + free(content->data); + retries--; + goto retrieve; + } + + break; + default: + log_err(_("Fetch failed with code %ld for URL %s"), + code, + url); + } } if (!result) @@ -116,50 +162,29 @@ static json_object *get_json_object(const char *url) return NULL; } -static void **list_add_list(void **list1, void **list2) -{ - int n1, n2, n; - void **list; - - n1 = list_length(list1); - n2 = list_length(list2); - - n = n1 + n2 + 1; - - list = malloc(sizeof(void *)*(n+1)); - - memcpy(list, list1, n1*sizeof(void *)); - memcpy(list+n1, list2, n2*sizeof(void *)); - - list[n1+n2] = NULL; - - free(list1); - - return list; -} - #define json_object_to_bpph_list \ json_object_to_binary_package_publishing_history_list struct binary_package_publishing_history * * get_binary_package_publishing_history_list(const char *archive_url, - const char *package_status) + const char *pkg_status) { struct json_object *o_next; - char *url = malloc(strlen(archive_url)+ - strlen(QUERY_GET_PUBLISHED_BINARIES)+ - strlen("&status=")+ - 9+ - 1); + char *url; json_object *o; void **result = NULL; + url = malloc(strlen(archive_url)+ + strlen(QUERY_GET_PUBLISHED_BINARIES)+ + (pkg_status ? strlen("&status=")+strlen(pkg_status) : 0)+ + 1); + strcpy(url, archive_url); strcat(url, QUERY_GET_PUBLISHED_BINARIES); - if (package_status) { + if (pkg_status) { strcat(url, "&status="); - strcat(url, package_status); + strcat(url, pkg_status); } while (url) { @@ -170,8 +195,8 @@ get_binary_package_publishing_history_list(const char *archive_url, if (!o) break; - result = list_add_list(result, - (void **)json_object_to_bpph_list(o)); + result = list_append_list(result, + (void **)json_object_to_bpph_list(o)); o_next = json_object_object_get(o, "next_collection_link"); @@ -211,12 +236,24 @@ const struct distro_arch_series *get_distro_arch_series(const char *url) { json_object *obj; const struct distro_arch_series *distro; + char *content; distro = cache_get(url); if (distro) return (struct distro_arch_series *)distro; - obj = get_json_object(url); + content = fcache_get(url + 7); + if (!content) { + content = fetch_url(url); + if (content) + fcache_put(url + 7, content); + else + return NULL; + } + + obj = json_tokener_parse(content); + + free(content); if (!obj) return NULL; @@ -279,8 +316,7 @@ struct daily_download_total **get_daily_download_totals(const char *binary_url) void lp_ws_cleanup() { - if (debug) - printf("DEBUG: cleanup CURL\n"); + log_debug(_("cleanup CURL")); curl_easy_cleanup(curl); curl_global_cleanup();