From: Jean-Philippe Orsini <jeanfi@gmail.com>
Date: Sun, 2 Mar 2014 20:50:08 +0000 (+0000)
Subject: split LP ddts requests when LP cannot retrieve all ddts with a uniq
X-Git-Tag: v1.3.0~3
X-Git-Url: https://git.wpitchoune.net/gitweb/?p=ppastats.git;a=commitdiff_plain;h=1a7e1ac9638c4e17a483b3f20886711679774a91

split LP ddts requests when LP cannot retrieve all ddts with a uniq
  request.
---

diff --git a/NEWS b/NEWS
index b47a9fa..c76514a 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,9 @@
+v1.3.0
+------
+
+* split LP ddts requests when LP cannot retrieve all ddts with a uniq
+  request.
+
 v1.2.0
 ------
 
diff --git a/configure.ac b/configure.ac
index 461609c..210f0a3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.64])
-AC_INIT([ppastats], [1.2.x],[jeanfi@gmail.com],[ppastats],[http://wpitchoune.net/ppastats])
+AC_INIT([ppastats], [1.3.x],[jeanfi@gmail.com],[ppastats],[http://wpitchoune.net/ppastats])
 
 AM_INIT_AUTOMAKE([-Wall gnu])
 
diff --git a/src/http.c b/src/http.c
index 99a6db3..dd99249 100644
--- a/src/http.c
+++ b/src/http.c
@@ -30,7 +30,7 @@
 #include "http.h"
 #include <plog.h>
 
-static const int DEFAULT_FETCH_RETRIES = 15;
+static const int DEFAULT_FETCH_RETRIES = 10;
 
 static CURL *curl;
 
diff --git a/src/lp_ws.c b/src/lp_ws.c
index 74d63c7..49166f4 100644
--- a/src/lp_ws.c
+++ b/src/lp_ws.c
@@ -352,10 +352,10 @@ static json_object *ddts_to_json_for_cache(struct daily_download_total **ddts)
 	return j_ddts;
 }
 
-char *create_ddts_query(const char *binary_url, time_t st)
+char *create_ddts_query(const char *binary_url, time_t st, time_t et)
 {
 	char *q;
-	char *sdate;
+	char *sdate, *edate;
 
 	if (st) {
 		sdate = time_to_ISO8601_date(&st);
@@ -363,12 +363,20 @@ char *create_ddts_query(const char *binary_url, time_t st)
 		q = malloc(strlen(binary_url)
 			   + strlen(QUERY_GET_DAILY_DOWNLOAD_TOTALS)
 			   + strlen("&start_date=YYYY-MM-DD")
+			   + strlen("&end_date=YYYY-MM-DD")
 			   + 1);
 		strcpy(q, binary_url);
 		strcat(q, QUERY_GET_DAILY_DOWNLOAD_TOTALS);
 		strcat(q, "&start_date=");
 		strcat(q, sdate);
 
+		if (et > 0) {
+			edate = time_to_ISO8601_date(&et);
+			strcat(q, "&end_date=");
+			strcat(q, edate);
+			free(edate);
+		}
+
 		free(sdate);
 	} else {
 		q = malloc(strlen(binary_url)
@@ -381,9 +389,61 @@ char *create_ddts_query(const char *binary_url, time_t st)
 	return q;
 }
 
-struct daily_download_total **get_daily_download_totals(const char *binary_url)
+static struct daily_download_total **retrieve_ddts(const char *binary_url,
+						   time_t date_since)
 {
-	char *url, *key, *content;
+	char *url;
+	json_object *json;
+	struct daily_download_total **ddts, **tmp;
+	time_t crt;
+
+	url = create_ddts_query(binary_url, date_since, 0);
+	json = get_json_object(url);
+	free(url);
+
+	if (json) {
+		ddts = json_object_to_daily_download_totals(json);
+		json_object_put(json);
+	} else {
+		crt = time(NULL);
+		ddts = NULL;
+
+		while (date_since < crt) {
+			url = create_ddts_query(binary_url, date_since, date_since);
+			json = get_json_object(url);
+			free(url);
+
+			if (!json)
+				break;
+
+			tmp = json_object_to_daily_download_totals(json);
+			json_object_put(json);
+			ddts = ddts_merge(ddts, tmp);
+			free(tmp);
+
+			date_since = date_since + 24 * 60 * 60; /* +1 day */
+
+			url = create_ddts_query(binary_url, date_since, 0);
+			json = get_json_object(url);
+			free(url);
+
+			if (json) {
+				tmp = json_object_to_daily_download_totals(json);
+				json_object_put(json);
+				ddts = ddts_merge(ddts, tmp);
+				free(tmp);
+				break;
+			}
+		}
+	}
+
+	return ddts;
+}
+
+struct daily_download_total **get_daily_download_totals(const char *binary_url,
+							time_t date_created)
+{
+	char *key, *content;
 	json_object *j_ddts, *json;
 	struct daily_download_total **retrieved_ddts = NULL;
 	struct daily_download_total **cached_ddts;
@@ -409,26 +469,20 @@ struct daily_download_total **get_daily_download_totals(const char *binary_url)
 		cached_ddts = NULL;
 	}
 
-	url = create_ddts_query(binary_url, last_t);
-
-	json = get_json_object(url);
-
-	free(url);
-
-	if (json) {
-		retrieved_ddts = json_object_to_daily_download_totals(json);
+	if (last_t > 0)
+		retrieved_ddts = retrieve_ddts(binary_url, last_t);
+	else
+		retrieved_ddts = retrieve_ddts(binary_url, date_created);
 
-		ddts = ddts_merge(cached_ddts, retrieved_ddts);
+	ddts = ddts_merge(cached_ddts, retrieved_ddts);
 
-		json_object_put(json);
+	if (ddts) {
 		j_ddts = ddts_to_json_for_cache(ddts);
 		fcache_put(key, json_object_get_string(j_ddts));
 		json_object_put(j_ddts);
-	} else {
-		ddts = NULL;
 	}
-
 	free(key);
+
 	if (ddts != cached_ddts)
 		daily_download_total_list_free(cached_ddts);
 	daily_download_total_list_free(retrieved_ddts);
diff --git a/src/lp_ws.h b/src/lp_ws.h
index b822ce9..83fb2f6 100644
--- a/src/lp_ws.h
+++ b/src/lp_ws.h
@@ -32,7 +32,8 @@ int get_download_count(const char *archive_url);
 
 const struct distro_arch_series *get_distro_arch_series(const char *url);
 
-struct daily_download_total **get_daily_download_totals(const char *binary_url);
+struct daily_download_total **get_daily_download_totals(const char *binary_url,
+							time_t date_created);
 
 const struct distro_series *get_distro_series(const char *distro_series_url);
 
diff --git a/src/ppastats.c b/src/ppastats.c
index a03ecf0..0990e0f 100644
--- a/src/ppastats.c
+++ b/src/ppastats.c
@@ -368,7 +368,7 @@ create_ppa_stats(const char *owner,
 
 	for (h_cur = history; *h_cur; ++h_cur) {
 		h = *h_cur;
-		totals = get_daily_download_totals(h->self_link);
+		totals = get_daily_download_totals(h->self_link, h->date_created);
 		if (!totals) {
 			log_err(_("Failed to retrieve download totals for %s"),
 				h->self_link);