X-Git-Url: http://git.wpitchoune.net/gitweb/?a=blobdiff_plain;f=src%2Flp_ws.c;h=3086e6f3960eb3c5bf7e3cfd1944e35e42a7f456;hb=c75862be119ac38a67f841fcc7da339be423973a;hp=d6f4087ffa60584cd8ab4f3a5912d66abf612904;hpb=d0059fa2611c9db88dd365b77ce49c04d2102de3;p=ppastats.git diff --git a/src/lp_ws.c b/src/lp_ws.c index d6f4087..3086e6f 100644 --- a/src/lp_ws.c +++ b/src/lp_ws.c @@ -1,156 +1,198 @@ /* - Copyright (C) 2011 jeanfi@gmail.com - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - 02110-1301 USA -*/ - + * Copyright (C) 2011-2012 jeanfi@gmail.com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA + */ + +#include +#define _(String) gettext(String) + +#include #include #include -#include #include #include "cache.h" +#include "fcache.h" +#include "http.h" #include "list.h" +#include "log.h" #include "lp_ws.h" #include "lp_json.h" #include "ppastats.h" -#define QUERY_GET_PUBLISHED_BINARIES \ - "?ws.op=getPublishedBinaries" -#define QUERY_GET_DOWNLOAD_COUNT "?ws.op=getDownloadCount" -#define QUERY_GET_DAILY_DOWNLOAD_TOTALS \ - "?ws.op=getDailyDownloadTotals" - -static CURL *curl; +/** Default ws.size value for the getPublishedBinaries request. */ +static const int DEFAULT_WS_SIZE = 150; -struct ucontent { - char *data; - size_t len; -}; +static const char *QUERY_GET_DOWNLOAD_COUNT = "?ws.op=getDownloadCount"; +static const char * +QUERY_GET_DAILY_DOWNLOAD_TOTALS = "?ws.op=getDailyDownloadTotals"; -static size_t cbk_curl(void *buffer, size_t size, size_t nmemb, void *userp) +static json_object *get_json_object(const char *url) { - size_t realsize = size * nmemb; - struct ucontent *mem = (struct ucontent *)userp; + json_object *obj = NULL; + char *body; - mem->data = realloc(mem->data, mem->len + realsize + 1); + body = get_url_content(url, 0); - memcpy(&(mem->data[mem->len]), buffer, realsize); - mem->len += realsize; - mem->data[mem->len] = 0; + if (body) { + obj = json_tokener_parse(body); + + free(body); + + return obj; + } - return realsize; + return NULL; } -static char *fetch_url(const char *url) +static char *get_bpph_list_cache_key(const char *archive_url) { - struct ucontent *content = malloc(sizeof(struct ucontent)); - char *result = NULL; - long code; + char *key; - if (debug) - printf("DEBUG: fetch_url %s\n", url); + key = malloc(strlen(archive_url + 7) + strlen("/bpph") + 1); + sprintf(key, "%s/bpph", archive_url + 7); - if (!curl) { - curl_global_init(CURL_GLOBAL_ALL); - curl = curl_easy_init(); - } + return key; +} - if (!curl) - exit(EXIT_FAILURE); +static struct bpph **get_bpph_list_from_cache(const char *key) +{ + char *content; + struct bpph **list; + json_object *json; - content->data = malloc(1); - content->data[0] = '\0'; - content->len = 0; + content = fcache_get(key); + if (!content) + return NULL; - curl_easy_setopt(curl, CURLOPT_URL, url); - curl_easy_setopt(curl, CURLOPT_VERBOSE, 0); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cbk_curl); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, content); - curl_easy_setopt(curl, CURLOPT_USERAGENT, "ppastats/0.0"); + json = json_tokener_parse(content); + if (!json) + return NULL; - if (curl_easy_perform(curl) == CURLE_OK) { + list = json_object_to_bpph_list(json); - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); - if (code == 200) - result = content->data; - } + json_object_put(json); + free(content); - if (!result) - free(content->data); + return list; +} - free(content); +static char *get_last_creation_date(struct bpph **list) +{ + time_t last, t; + struct bpph **cur; - return result; + last = 0; + + if (list) + for (cur = list; *cur; cur++) { + t = (*cur)->date_created; + if (t > last) + last = t; + } + + if (last) + return time_to_str(last); + else + return NULL; } -static json_object *get_json_object(const char *url) +/* + * 'archive_url': LP URL of the archive. + * 'size': size of the reply array. Between 1-300, else default value is used. + */ +static char *create_query_get_bpph(const char *archive_url, + const char *status, + int size) { - json_object *obj = NULL; - char *body; + static const char *default_opt = "?ws.op=getPublishedBinaries&ws.size="; + static const char *status_opt = "&status="; + char *url; + size_t n; - body = fetch_url(url); + if (size < 1 || size > 300) + size = DEFAULT_WS_SIZE; - if (body) { - obj = json_tokener_parse(body); + n = strlen(archive_url) + strlen(default_opt) + 3 + 1; - free(body); + if (status) + n += strlen(status_opt) + strlen(status); - return obj; + url = malloc(n); + sprintf(url, "%s%s%d", archive_url, default_opt, size); + + if (status) { + strcat(url, status_opt); + strcat(url, status); } - return NULL; + return url; } -#define json_object_to_bpph_list \ -json_object_to_binary_package_publishing_history_list - -struct binary_package_publishing_history * * -get_binary_package_publishing_history_list(const char *archive_url, - const char *pkg_status) +struct bpph **get_bpph_list(const char *archive_url, + const char *pkg_status, + int ws_size) { - struct json_object *o_next; - char *url; - json_object *o; - void **result = NULL; + char *url, *key, *tmp; + struct bpph **result; + struct json_object *o, *bpph_json, *o_next; + char *date; + int ok; - url = malloc(strlen(archive_url)+ - strlen(QUERY_GET_PUBLISHED_BINARIES)+ - (pkg_status ? strlen("&status=")+strlen(pkg_status) : 0)+ - 1); + url = create_query_get_bpph(archive_url, pkg_status, ws_size); - strcpy(url, archive_url); - strcat(url, QUERY_GET_PUBLISHED_BINARIES); + key = get_bpph_list_cache_key(archive_url); + + result = get_bpph_list_from_cache(key); + + if (result) { + date = get_last_creation_date(result); + + if (date) { + printf("Update package since: %s\n", date); - if (pkg_status) { - strcat(url, "&status="); - strcat(url, pkg_status); + tmp = malloc(strlen(url) + + strlen("&created_since_date=") + + strlen(date)+1); + strcpy(tmp, url); + strcat(tmp, "&created_since_date="); + strcat(tmp, date); + + free(url); + url = tmp; + + free(date); + } } + ok = 1; while (url) { o = get_json_object(url); free(url); url = NULL; - if (!o) + if (!o) { + ok = 0; break; + } - result = list_add_list(result, - (void **)json_object_to_bpph_list(o)); + result = bpph_list_append_list(result, + json_object_to_bpph_list(o)); o_next = json_object_object_get(o, "next_collection_link"); @@ -160,7 +202,15 @@ get_binary_package_publishing_history_list(const char *archive_url, json_object_put(o); } - return (struct binary_package_publishing_history **)result; + if (ok) { + bpph_json = bpph_list_to_json(result); + fcache_put(key, json_object_to_json_string(bpph_json)); + json_object_put(bpph_json); + } + + free(key); + + return result; } int get_download_count(const char *archive_url) @@ -190,12 +240,20 @@ const struct distro_arch_series *get_distro_arch_series(const char *url) { json_object *obj; const struct distro_arch_series *distro; + char *content; distro = cache_get(url); if (distro) return (struct distro_arch_series *)distro; - obj = get_json_object(url); + content = get_url_content(url, 1); + + if (!content) + return NULL; + + obj = json_tokener_parse(content); + + free(content); if (!obj) return NULL; @@ -213,12 +271,20 @@ const struct distro_series *get_distro_series(const char *url) { json_object *obj; const struct distro_series *distro; + char *content; distro = cache_get(url); if (distro) return (struct distro_series *)distro; - obj = get_json_object(url); + content = get_url_content(url, 1); + + if (!content) + return NULL; + + obj = json_tokener_parse(content); + + free(content); if (!obj) return NULL; @@ -256,11 +322,3 @@ struct daily_download_total **get_daily_download_totals(const char *binary_url) return result; } -void lp_ws_cleanup() -{ - if (debug) - printf("DEBUG: cleanup CURL\n"); - - curl_easy_cleanup(curl); - curl_global_cleanup(); -}