X-Git-Url: https://git.wpitchoune.net/gitweb/?a=blobdiff_plain;f=src%2Flp_ws.c;h=74bcde5866c3e8224743a772053e4b464126cf7c;hb=12b0e2c57c55b4d3086ef920deda5400016339f2;hp=20443716c0790ffc3f9d34d7e5f1782d667c232d;hpb=967d20c38c65689c96a0276e6f916b4f8e2617de;p=ppastats.git diff --git a/src/lp_ws.c b/src/lp_ws.c index 2044371..74bcde5 100644 --- a/src/lp_ws.c +++ b/src/lp_ws.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2012 jeanfi@gmail.com + * Copyright (C) 2011-2014 jeanfi@gmail.com * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -20,22 +20,27 @@ #include #define _(String) gettext(String) +#include #include #include +#include +#include -#include +#include #include "cache.h" #include "fcache.h" #include "http.h" #include "list.h" -#include "log.h" #include "lp_ws.h" #include "lp_json.h" +#include #include "ppastats.h" +#include + +/** Default ws.size value for the getPublishedBinaries request. */ +static const int DEFAULT_WS_SIZE = 150; -static const char * -QUERY_GET_PUBLISHED_BINARIES = "?ws.op=getPublishedBinaries&ws.size=150"; static const char *QUERY_GET_DOWNLOAD_COUNT = "?ws.op=getDownloadCount"; static const char * QUERY_GET_DAILY_DOWNLOAD_TOTALS = "?ws.op=getDailyDownloadTotals"; @@ -58,36 +63,147 @@ static json_object *get_json_object(const char *url) return NULL; } -struct bpph **get_bpph_list(const char *archive_url, const char *pkg_status) +static char *get_bpph_list_cache_key(const char *archive_url) { - struct json_object *o_next; + char *key; + + key = malloc(strlen(archive_url + 7) + strlen("/bpph") + 1); + sprintf(key, "%s/bpph", archive_url + 7); + + return key; +} + +static char *get_ddts_list_cache_key(const char *url) +{ + char *key; + + key = malloc(strlen(url + 7) + strlen("/ddts") + 1); + sprintf(key, "%s/ddts", url + 7); + + return key; +} + +static struct bpph **get_bpph_list_from_cache(const char *key) +{ + char *content; + struct bpph **list; + json_object *json; + + content = fcache_get(key); + if (!content) + return NULL; + + json = json_tokener_parse(content); + if (!json) + return NULL; + + list = json_object_to_bpph_list(json); + + json_object_put(json); + free(content); + + return list; +} + +static char *get_last_creation_date(struct bpph **list) +{ + time_t last, t; + struct bpph **cur; + + last = 0; + + if (list) + for (cur = list; *cur; cur++) { + t = (*cur)->date_created; + if (t > last) + last = t; + } + + if (last) + return time_to_ISO8601_time(&last); + else + return NULL; +} + +/* + * 'archive_url': LP URL of the archive. + * 'size': size of the reply array. Between 1-300, else default value is used. + */ +static char *create_query_get_bpph(const char *archive_url, + const char *status, + int size) +{ + static const char *default_opt = "?ws.op=getPublishedBinaries&ws.size="; + static const char *status_opt = "&status="; char *url; - json_object *o; - void **result = NULL; + size_t n; - url = malloc(strlen(archive_url)+ - strlen(QUERY_GET_PUBLISHED_BINARIES)+ - (pkg_status ? strlen("&status=")+strlen(pkg_status) : 0)+ - 1); + if (size < 1 || size > 300) + size = DEFAULT_WS_SIZE; - strcpy(url, archive_url); - strcat(url, QUERY_GET_PUBLISHED_BINARIES); + n = strlen(archive_url) + strlen(default_opt) + 3 + 1; + + if (status) + n += strlen(status_opt) + strlen(status); + + url = malloc(n); + sprintf(url, "%s%s%d", archive_url, default_opt, size); + + if (status) { + strcat(url, status_opt); + strcat(url, status); + } + + return url; +} - if (pkg_status) { - strcat(url, "&status="); - strcat(url, pkg_status); +struct bpph **get_bpph_list(const char *archive_url, + const char *pkg_status, + int ws_size) +{ + char *url, *key, *tmp; + struct bpph **result; + struct json_object *o, *bpph_json, *o_next; + char *date; + int ok; + + url = create_query_get_bpph(archive_url, pkg_status, ws_size); + + key = get_bpph_list_cache_key(archive_url); + + result = get_bpph_list_from_cache(key); + + if (result) { + date = get_last_creation_date(result); + + if (date) { + tmp = malloc(strlen(url) + + strlen("&created_since_date=") + + strlen(date)+1); + strcpy(tmp, url); + strcat(tmp, "&created_since_date="); + strcat(tmp, date); + + free(url); + url = tmp; + + free(date); + } } + ok = 1; while (url) { o = get_json_object(url); free(url); url = NULL; - if (!o) + if (!o) { + ok = 0; break; + } - result = list_append_list(result, - (void **)json_object_to_bpph_list(o)); + result = bpph_list_append_list(result, + json_object_to_bpph_list(o)); o_next = json_object_object_get(o, "next_collection_link"); @@ -97,7 +213,15 @@ struct bpph **get_bpph_list(const char *archive_url, const char *pkg_status) json_object_put(o); } - return (struct bpph **)result; + if (ok) { + bpph_json = bpph_list_to_json(result); + fcache_put(key, json_object_to_json_string(bpph_json)); + json_object_put(bpph_json); + } + + free(key); + + return result; } int get_download_count(const char *archive_url) @@ -185,27 +309,126 @@ const struct distro_series *get_distro_series(const char *url) return distro; } -struct daily_download_total **get_daily_download_totals(const char *binary_url) +/* + Convert ddts older than 4 weeks to the same JSON representation than + the LP one. Newer ddts are not stored in the cache because the data + may change during following days. It avoids to miss downloads which + are not yet taken in consideration by LP. + */ +static json_object *ddts_to_json_for_cache(struct daily_download_total **ddts) { - char *url; - json_object *obj; - struct daily_download_total **result = NULL; + json_object *j_ddts; + struct daily_download_total *ddt; + char *date; + struct timeval *tv; + time_t t; + double d; - url = malloc(strlen(binary_url)+ - strlen(QUERY_GET_DAILY_DOWNLOAD_TOTALS)+1); + j_ddts = json_object_new_object(); - strcpy(url, binary_url); - strcat(url, QUERY_GET_DAILY_DOWNLOAD_TOTALS); + tv = malloc(sizeof(struct timeval)); + gettimeofday(tv, NULL); - obj = get_json_object(url); + while (ddts && *ddts) { + ddt = *ddts; + + t = mktime(&(ddt->date)); + + d = difftime(tv->tv_sec, t); + + if (d > 4 * 7 * 24 * 60 * 60) { /* older than 4 weeks */ + date = tm_to_ISO8601_date(&ddt->date); + json_object_object_add(j_ddts, + date, + json_object_new_int(ddt->count)); + free(date); + } - if (obj) { - result = json_object_to_daily_download_totals(obj); - json_object_put(obj); + ddts++; } + free(tv); + + return j_ddts; +} + +char *create_ddts_query(const char *binary_url, time_t st) +{ + char *q; + char *sdate; + + if (st) { + sdate = time_to_ISO8601_date(&st); + + q = malloc(strlen(binary_url) + + strlen(QUERY_GET_DAILY_DOWNLOAD_TOTALS) + + strlen("&start_date=YYYY-MM-DD") + + 1); + strcpy(q, binary_url); + strcat(q, QUERY_GET_DAILY_DOWNLOAD_TOTALS); + strcat(q, "&start_date="); + strcat(q, sdate); + + free(sdate); + } else { + q = malloc(strlen(binary_url) + + strlen(QUERY_GET_DAILY_DOWNLOAD_TOTALS) + + 1); + strcpy(q, binary_url); + strcat(q, QUERY_GET_DAILY_DOWNLOAD_TOTALS); + } + + return q; +} + +struct daily_download_total **get_daily_download_totals(const char *binary_url) +{ + char *url, *key, *content; + json_object *j_ddts, *json; + struct daily_download_total **retrieved_ddts = NULL; + struct daily_download_total **cached_ddts; + struct daily_download_total **ddts; + time_t last_t; + + key = get_ddts_list_cache_key(binary_url); + + content = fcache_get(key); + if (content) + json = json_tokener_parse(content); + else + json = NULL; + + if (json) { + cached_ddts = json_object_to_daily_download_totals(json); + last_t = ddts_get_last_date(cached_ddts); + } else { + last_t = 0; + cached_ddts = NULL; + } + + url = create_ddts_query(binary_url, last_t); + + json = get_json_object(url); + free(url); - return result; + if (json) { + retrieved_ddts = json_object_to_daily_download_totals(json); + + ddts = ddts_merge(cached_ddts, retrieved_ddts); + + json_object_put(json); + j_ddts = ddts_to_json_for_cache(ddts); + fcache_put(key, json_object_get_string(j_ddts)); + json_object_put(j_ddts); + } else { + ddts = NULL; + } + + free(key); + free(cached_ddts); + free(retrieved_ddts); + + return ddts; }