X-Git-Url: http://git.wpitchoune.net/gitweb/?a=blobdiff_plain;f=src%2Flp_ws.c;h=f04f16984b6239f504d749bd0b2abaf7bad9a747;hb=586361f5da21ef8e34987db0c628af1fb06df6a1;hp=3086e6f3960eb3c5bf7e3cfd1944e35e42a7f456;hpb=c75862be119ac38a67f841fcc7da339be423973a;p=ppastats.git diff --git a/src/lp_ws.c b/src/lp_ws.c index 3086e6f..f04f169 100644 --- a/src/lp_ws.c +++ b/src/lp_ws.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2012 jeanfi@gmail.com + * Copyright (C) 2011-2014 jeanfi@gmail.com * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -23,17 +23,20 @@ #include #include #include +#include +#include -#include +#include -#include "cache.h" -#include "fcache.h" -#include "http.h" -#include "list.h" -#include "log.h" -#include "lp_ws.h" -#include "lp_json.h" -#include "ppastats.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include /** Default ws.size value for the getPublishedBinaries request. */ static const int DEFAULT_WS_SIZE = 150; @@ -70,6 +73,16 @@ static char *get_bpph_list_cache_key(const char *archive_url) return key; } +static char *get_ddts_list_cache_key(const char *url) +{ + char *key; + + key = malloc(strlen(url + 7) + strlen("/ddts") + 1); + sprintf(key, "%s/ddts", url + 7); + + return key; +} + static struct bpph **get_bpph_list_from_cache(const char *key) { char *content; @@ -107,7 +120,7 @@ static char *get_last_creation_date(struct bpph **list) } if (last) - return time_to_str(last); + return time_to_ISO8601_time(&last); else return NULL; } @@ -164,8 +177,6 @@ struct bpph **get_bpph_list(const char *archive_url, date = get_last_creation_date(result); if (date) { - printf("Update package since: %s\n", date); - tmp = malloc(strlen(url) + strlen("&created_since_date=") + strlen(date)+1); @@ -298,27 +309,187 @@ const struct distro_series *get_distro_series(const char *url) return distro; } -struct daily_download_total **get_daily_download_totals(const char *binary_url) +/* + Convert ddts older than 4 weeks to the same JSON representation than + the LP one. Newer ddts are not stored in the cache because the data + may change during following days. It avoids to miss downloads which + are not yet taken in consideration by LP. + */ +static json_object *ddts_to_json_for_cache(struct daily_download_total **ddts) { - char *url; - json_object *obj; - struct daily_download_total **result = NULL; + json_object *j_ddts; + struct daily_download_total *ddt; + char *date; + struct timeval *tv; + time_t t; + double d; - url = malloc(strlen(binary_url)+ - strlen(QUERY_GET_DAILY_DOWNLOAD_TOTALS)+1); + j_ddts = json_object_new_object(); - strcpy(url, binary_url); - strcat(url, QUERY_GET_DAILY_DOWNLOAD_TOTALS); + tv = malloc(sizeof(struct timeval)); + gettimeofday(tv, NULL); - obj = get_json_object(url); + while (ddts && *ddts) { + ddt = *ddts; + + t = mktime(&(ddt->date)); + + d = difftime(tv->tv_sec, t); - if (obj) { - result = json_object_to_daily_download_totals(obj); - json_object_put(obj); + if (d > 4 * 7 * 24 * 60 * 60) { /* older than 4 weeks */ + date = tm_to_ISO8601_date(&ddt->date); + json_object_object_add(j_ddts, + date, + json_object_new_int(ddt->count)); + free(date); + } + + ddts++; } + free(tv); + + return j_ddts; +} + +char *create_ddts_query(const char *binary_url, time_t st, time_t et) +{ + char *q; + char *sdate, *edate; + + if (st) { + sdate = time_to_ISO8601_date(&st); + + q = malloc(strlen(binary_url) + + strlen(QUERY_GET_DAILY_DOWNLOAD_TOTALS) + + strlen("&start_date=YYYY-MM-DD") + + strlen("&end_date=YYYY-MM-DD") + + 1); + strcpy(q, binary_url); + strcat(q, QUERY_GET_DAILY_DOWNLOAD_TOTALS); + strcat(q, "&start_date="); + strcat(q, sdate); + + if (et > 0) { + edate = time_to_ISO8601_date(&et); + strcat(q, "&end_date="); + strcat(q, edate); + free(edate); + } + + free(sdate); + } else { + q = malloc(strlen(binary_url) + + strlen(QUERY_GET_DAILY_DOWNLOAD_TOTALS) + + 1); + strcpy(q, binary_url); + strcat(q, QUERY_GET_DAILY_DOWNLOAD_TOTALS); + } + + return q; +} + +static struct daily_download_total **retrieve_ddts(const char *binary_url, + time_t date_since) +{ + char *url; + json_object *json; + struct daily_download_total **ddts, **tmp; + time_t crt; + + url = create_ddts_query(binary_url, date_since, 0); + json = get_json_object(url); free(url); - return result; + if (json) { + ddts = json_object_to_daily_download_totals(json); + json_object_put(json); + } else { + crt = time(NULL); + ddts = NULL; + + while (date_since < crt) { + url = create_ddts_query(binary_url, + date_since, + date_since); + json = get_json_object(url); + free(url); + + if (!json) + break; + + tmp = json_object_to_daily_download_totals(json); + json_object_put(json); + ddts = ddts_merge(ddts, tmp); + free(tmp); + + date_since = date_since + 24 * 60 * 60; /* +1 day */ + + url = create_ddts_query(binary_url, date_since, 0); + json = get_json_object(url); + free(url); + + if (json) { + tmp = json_object_to_daily_download_totals + (json); + json_object_put(json); + ddts = ddts_merge(ddts, tmp); + free(tmp); + break; + } + } + } + + return ddts; +} + +struct daily_download_total **get_daily_download_totals(const char *binary_url, + time_t date_created) +{ + char *key, *content; + json_object *j_ddts, *json; + struct daily_download_total **retrieved_ddts = NULL; + struct daily_download_total **cached_ddts; + struct daily_download_total **ddts; + time_t last_t; + + key = get_ddts_list_cache_key(binary_url); + + content = fcache_get(key); + if (content) { + json = json_tokener_parse(content); + free(content); + } else { + json = NULL; + } + + if (json) { + cached_ddts = json_object_to_daily_download_totals(json); + json_object_put(json); + last_t = ddts_get_last_date(cached_ddts); + } else { + last_t = 0; + cached_ddts = NULL; + } + + if (last_t > 0) + retrieved_ddts = retrieve_ddts(binary_url, last_t); + else + retrieved_ddts = retrieve_ddts(binary_url, date_created); + + ddts = ddts_merge(cached_ddts, retrieved_ddts); + + if (ddts) { + j_ddts = ddts_to_json_for_cache(ddts); + fcache_put(key, json_object_get_string(j_ddts)); + json_object_put(j_ddts); + } + free(key); + + if (ddts != cached_ddts) + daily_download_total_list_free(cached_ddts); + daily_download_total_list_free(retrieved_ddts); + + return ddts; }