X-Git-Url: https://git.wpitchoune.net/gitweb/?a=blobdiff_plain;f=src%2Flp_ws.c;h=11092c4e3228566928ab8addb49735049eacb4fe;hb=a9435991d03d0bd9e81d6cb6003013bec4b2e6e9;hp=3de408b6ad7095b8991c50fafd494aa9d9354bad;hpb=301be453975067183367ea8c29069476bf1b45ff;p=ppastats.git diff --git a/src/lp_ws.c b/src/lp_ws.c index 3de408b..11092c4 100644 --- a/src/lp_ws.c +++ b/src/lp_ws.c @@ -20,155 +20,82 @@ #include #define _(String) gettext(String) +#include #include #include -#include -#include #include #include "cache.h" +#include "fcache.h" +#include "http.h" #include "list.h" #include "log.h" #include "lp_ws.h" #include "lp_json.h" #include "ppastats.h" -static const char *QUERY_GET_PUBLISHED_BINARIES = "?ws.op=getPublishedBinaries"; +static const char * +QUERY_GET_PUBLISHED_BINARIES = "?ws.op=getPublishedBinaries&ws.size=150"; static const char *QUERY_GET_DOWNLOAD_COUNT = "?ws.op=getDownloadCount"; static const char * QUERY_GET_DAILY_DOWNLOAD_TOTALS = "?ws.op=getDailyDownloadTotals"; -static const int DEFAULT_FETCH_RETRIES = 3; - -static CURL *curl; - -struct ucontent { - char *data; - size_t len; -}; - -static size_t cbk_curl(void *buffer, size_t size, size_t nmemb, void *userp) +static json_object *get_json_object(const char *url) { - size_t realsize = size * nmemb; - struct ucontent *mem = (struct ucontent *)userp; + json_object *obj = NULL; + char *body; - mem->data = realloc(mem->data, mem->len + realsize + 1); + body = get_url_content(url, 0); - memcpy(&(mem->data[mem->len]), buffer, realsize); - mem->len += realsize; - mem->data[mem->len] = 0; + if (body) { + obj = json_tokener_parse(body); - return realsize; -} + free(body); -static void init() -{ - if (!curl) { - log_debug(_("initializing CURL")); - curl_global_init(CURL_GLOBAL_ALL); - curl = curl_easy_init(); + return obj; } - if (!curl) - exit(EXIT_FAILURE); + return NULL; } -static char *fetch_url(const char *url) +static char *get_bpph_list_cache_key(const char *archive_url) { - struct ucontent *content = malloc(sizeof(struct ucontent)); - char *result; - long code; - int retries; - - log_debug(_("fetch_url(): %s"), url); - - init(); + char *key; - result = NULL; + key = malloc(strlen(archive_url + 7) + strlen("/bpph") + 1); + sprintf(key, "%s/bpph", archive_url + 7); - retries = DEFAULT_FETCH_RETRIES; - - retrieve: - content->data = malloc(1); - content->data[0] = '\0'; - content->len = 0; - - curl_easy_setopt(curl, CURLOPT_URL, url); - curl_easy_setopt(curl, CURLOPT_VERBOSE, 0); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cbk_curl); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, content); - curl_easy_setopt(curl, CURLOPT_USERAGENT, "ppastats/0.0"); - - if (curl_easy_perform(curl) == CURLE_OK) { - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); - - switch (code) { - case 200: - result = content->data; - break; - case 500: - case 502: - case 503: - case 504: - log_err(_("Fetch failed with code %ld for URL %s"), - code, - url); - - if (retries) { - log_debug(_("Wait 5s before retry")); - sleep(5); - - free(content->data); - retries--; - goto retrieve; - } - - break; - default: - log_err(_("Fetch failed with code %ld for URL %s"), - code, - url); - } - } - - if (!result) - free(content->data); - - free(content); - - return result; + return key; } -static json_object *get_json_object(const char *url) +static struct bpph **get_bpph_list_from_cache(const char *key) { - json_object *obj = NULL; - char *body; + char *content; + struct bpph **list; + json_object *json; - body = fetch_url(url); + content = fcache_get(key); + if (!content) + return NULL; - if (body) { - obj = json_tokener_parse(body); + json = json_tokener_parse(content); + if (!json) + return NULL; - free(body); + list = json_object_to_bpph_list(json); - return obj; - } + json_object_put(json); + free(content); - return NULL; + return list; } -#define json_object_to_bpph_list \ -json_object_to_binary_package_publishing_history_list - -struct binary_package_publishing_history * * -get_binary_package_publishing_history_list(const char *archive_url, - const char *pkg_status) +struct bpph **get_bpph_list(const char *archive_url, const char *pkg_status) { - struct json_object *o_next; - char *url; - json_object *o; - void **result = NULL; + char *url, *key; + struct bpph **result = NULL; + struct json_object *o, *bpph_json, *o_next; url = malloc(strlen(archive_url)+ strlen(QUERY_GET_PUBLISHED_BINARIES)+ @@ -183,6 +110,9 @@ get_binary_package_publishing_history_list(const char *archive_url, strcat(url, pkg_status); } + key = get_bpph_list_cache_key(archive_url); + result = get_bpph_list_from_cache(key); + while (url) { o = get_json_object(url); free(url); @@ -191,8 +121,8 @@ get_binary_package_publishing_history_list(const char *archive_url, if (!o) break; - result = list_append_list(result, - (void **)json_object_to_bpph_list(o)); + result = bpph_list_append_list(result, + json_object_to_bpph_list(o)); o_next = json_object_object_get(o, "next_collection_link"); @@ -202,7 +132,14 @@ get_binary_package_publishing_history_list(const char *archive_url, json_object_put(o); } - return (struct binary_package_publishing_history **)result; + bpph_json = bpph_list_to_json(result); + + fcache_put(key, json_object_to_json_string(bpph_json)); + + json_object_put(bpph_json); + free(key); + + return result; } int get_download_count(const char *archive_url) @@ -232,12 +169,20 @@ const struct distro_arch_series *get_distro_arch_series(const char *url) { json_object *obj; const struct distro_arch_series *distro; + char *content; distro = cache_get(url); if (distro) return (struct distro_arch_series *)distro; - obj = get_json_object(url); + content = get_url_content(url, 1); + + if (!content) + return NULL; + + obj = json_tokener_parse(content); + + free(content); if (!obj) return NULL; @@ -255,12 +200,20 @@ const struct distro_series *get_distro_series(const char *url) { json_object *obj; const struct distro_series *distro; + char *content; distro = cache_get(url); if (distro) return (struct distro_series *)distro; - obj = get_json_object(url); + content = get_url_content(url, 1); + + if (!content) + return NULL; + + obj = json_tokener_parse(content); + + free(content); if (!obj) return NULL; @@ -298,10 +251,3 @@ struct daily_download_total **get_daily_download_totals(const char *binary_url) return result; } -void lp_ws_cleanup() -{ - log_debug(_("cleanup CURL")); - - curl_easy_cleanup(curl); - curl_global_cleanup(); -}