split LP ddts requests when LP cannot retrieve all ddts with a uniq

  request.

diff --git a/NEWS b/NEWS
index b47a9fa..c76514a 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,9 @@
+v1.3.0
+------
+
+* split LP ddts requests when LP cannot retrieve all ddts with a uniq
+  request.
+
 v1.2.0
 ------
 

diff --git a/configure.ac b/configure.ac
index 461609c..210f0a3 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.64])
-AC_INIT([ppastats], [1.2.x],[jeanfi@gmail.com],[ppastats],[http://wpitchoune.net/ppastats])
+AC_INIT([ppastats], [1.3.x],[jeanfi@gmail.com],[ppastats],[http://wpitchoune.net/ppastats])
 
 AM_INIT_AUTOMAKE([-Wall gnu])
 

diff --git a/src/http.c b/src/http.c
index 99a6db3..dd99249 100644 (file)
--- a/src/http.c
+++ b/src/http.c
@@ -30,7 +30,7 @@
 #include "http.h"
 #include <plog.h>
 
-static const int DEFAULT_FETCH_RETRIES = 15;
+static const int DEFAULT_FETCH_RETRIES = 10;
 
 static CURL *curl;
 

diff --git a/src/lp_ws.c b/src/lp_ws.c
index 74d63c7..49166f4 100644 (file)
--- a/src/lp_ws.c
+++ b/src/lp_ws.c
@@ -352,10 +352,10 @@ static json_object *ddts_to_json_for_cache(struct daily_download_total **ddts)
        return j_ddts;
 }
 
-char *create_ddts_query(const char *binary_url, time_t st)
+char *create_ddts_query(const char *binary_url, time_t st, time_t et)
 {
        char *q;
-       char *sdate;
+       char *sdate, *edate;
 
        if (st) {
                sdate = time_to_ISO8601_date(&st);
@@ -363,12 +363,20 @@ char *create_ddts_query(const char *binary_url, time_t st)
                q = malloc(strlen(binary_url)
                           + strlen(QUERY_GET_DAILY_DOWNLOAD_TOTALS)
                           + strlen("&start_date=YYYY-MM-DD")
+                          + strlen("&end_date=YYYY-MM-DD")
                           + 1);
                strcpy(q, binary_url);
                strcat(q, QUERY_GET_DAILY_DOWNLOAD_TOTALS);
                strcat(q, "&start_date=");
                strcat(q, sdate);
 
+               if (et > 0) {
+                       edate = time_to_ISO8601_date(&et);
+                       strcat(q, "&end_date=");
+                       strcat(q, edate);
+                       free(edate);
+               }
+
                free(sdate);
        } else {
                q = malloc(strlen(binary_url)
@@ -381,9 +389,61 @@ char *create_ddts_query(const char *binary_url, time_t st)
        return q;
 }
 
-struct daily_download_total **get_daily_download_totals(const char *binary_url)
+static struct daily_download_total **retrieve_ddts(const char *binary_url,
+                                                  time_t date_since)
 {
-       char *url, *key, *content;
+       char *url;
+       json_object *json;
+       struct daily_download_total **ddts, **tmp;
+       time_t crt;
+
+       url = create_ddts_query(binary_url, date_since, 0);
+       json = get_json_object(url);
+       free(url);
+
+       if (json) {
+               ddts = json_object_to_daily_download_totals(json);
+               json_object_put(json);
+       } else {
+               crt = time(NULL);
+               ddts = NULL;
+
+               while (date_since < crt) {
+                       url = create_ddts_query(binary_url, date_since, date_since);
+                       json = get_json_object(url);
+                       free(url);
+
+                       if (!json)
+                               break;
+
+                       tmp = json_object_to_daily_download_totals(json);
+                       json_object_put(json);
+                       ddts = ddts_merge(ddts, tmp);
+                       free(tmp);
+
+                       date_since = date_since + 24 * 60 * 60; /* +1 day */
+
+                       url = create_ddts_query(binary_url, date_since, 0);
+                       json = get_json_object(url);
+                       free(url);
+
+                       if (json) {
+                               tmp = json_object_to_daily_download_totals(json);
+                               json_object_put(json);
+                               ddts = ddts_merge(ddts, tmp);
+                               free(tmp);
+                               break;
+                       }
+               }
+       }
+
+       return ddts;
+}
+
+struct daily_download_total **get_daily_download_totals(const char *binary_url,
+                                                       time_t date_created)
+{
+       char *key, *content;
        json_object *j_ddts, *json;
        struct daily_download_total **retrieved_ddts = NULL;
        struct daily_download_total **cached_ddts;
@@ -409,26 +469,20 @@ struct daily_download_total **get_daily_download_totals(const char *binary_url)
                cached_ddts = NULL;
        }
 
-       url = create_ddts_query(binary_url, last_t);
-
-       json = get_json_object(url);
-
-       free(url);
-
-       if (json) {
-               retrieved_ddts = json_object_to_daily_download_totals(json);
+       if (last_t > 0)
+               retrieved_ddts = retrieve_ddts(binary_url, last_t);
+       else
+               retrieved_ddts = retrieve_ddts(binary_url, date_created);
 
-               ddts = ddts_merge(cached_ddts, retrieved_ddts);
+       ddts = ddts_merge(cached_ddts, retrieved_ddts);
 
-               json_object_put(json);
+       if (ddts) {
                j_ddts = ddts_to_json_for_cache(ddts);
                fcache_put(key, json_object_get_string(j_ddts));
                json_object_put(j_ddts);
-       } else {
-               ddts = NULL;
        }
-
        free(key);
+
        if (ddts != cached_ddts)
                daily_download_total_list_free(cached_ddts);
        daily_download_total_list_free(retrieved_ddts);

diff --git a/src/lp_ws.h b/src/lp_ws.h
index b822ce9..83fb2f6 100644 (file)
--- a/src/lp_ws.h
+++ b/src/lp_ws.h
@@ -32,7 +32,8 @@ int get_download_count(const char *archive_url);
 
 const struct distro_arch_series *get_distro_arch_series(const char *url);
 
-struct daily_download_total **get_daily_download_totals(const char *binary_url);
+struct daily_download_total **get_daily_download_totals(const char *binary_url,
+                                                       time_t date_created);
 
 const struct distro_series *get_distro_series(const char *distro_series_url);
 

diff --git a/src/ppastats.c b/src/ppastats.c
index a03ecf0..0990e0f 100644 (file)
--- a/src/ppastats.c
+++ b/src/ppastats.c
@@ -368,7 +368,7 @@ create_ppa_stats(const char *owner,
 
        for (h_cur = history; *h_cur; ++h_cur) {
                h = *h_cur;
-               totals = get_daily_download_totals(h->self_link);
+               totals = get_daily_download_totals(h->self_link, h->date_created);
                if (!totals) {
                        log_err(_("Failed to retrieve download totals for %s"),
                                h->self_link);