X-Git-Url: http://git.wpitchoune.net/gitweb/?a=blobdiff_plain;f=src%2Flp_ws.c;h=b6957eb83314a62ffb6d07b67be9c831a7473944;hb=9d8ca122223ad917f61d4f8f5e32f809b5926222;hp=a6b95d11bb1f346135df750e851fb2304e97fc99;hpb=5ed73c995d6fcb5f3b4626bf7ebf609fc52f5d94;p=ppastats.git

diff --git a/src/lp_ws.c b/src/lp_ws.c
index a6b95d1..b6957eb 100644
--- a/src/lp_ws.c
+++ b/src/lp_ws.c
@@ -19,19 +19,25 @@
 
 #include <stdlib.h>
 #include <string.h>
+#include <unistd.h>
 
 #include <curl/curl.h>
 #include <json/json.h>
 
+#include "cache.h"
+#include "list.h"
 #include "lp_ws.h"
 #include "lp_json.h"
+#include "ppastats.h"
 
 #define QUERY_GET_PUBLISHED_BINARIES \
-	"?ws.op=getPublishedBinaries&status=Published"
+	"?ws.op=getPublishedBinaries"
 #define QUERY_GET_DOWNLOAD_COUNT "?ws.op=getDownloadCount"
 #define QUERY_GET_DAILY_DOWNLOAD_TOTALS \
 	"?ws.op=getDailyDownloadTotals"
 
+static int DEFAULT_FETCH_RETRIES = 3;
+
 static CURL *curl;
 
 struct ucontent {
@@ -56,15 +62,28 @@ static size_t cbk_curl(void *buffer, size_t size, size_t nmemb, void *userp)
 static char *fetch_url(const char *url)
 {
 	struct ucontent *content = malloc(sizeof(struct ucontent));
-	char *result = NULL;
+	char *result;
 	long code;
+	int retries;
 
-	if (!curl)
+	if (debug)
+		printf("DEBUG: fetch_url %s\n", url);
+
+	if (!curl) {
+		if (debug)
+			printf("DEBUG: initializing CURL\n");
+		curl_global_init(CURL_GLOBAL_ALL);
 		curl = curl_easy_init();
+	}
 
 	if (!curl)
 		exit(EXIT_FAILURE);
 
+	result = NULL;
+
+	retries = DEFAULT_FETCH_RETRIES;
+
+ retrieve:
 	content->data = malloc(1);
 	content->data[0] = '\0';
 	content->len = 0;
@@ -76,10 +95,30 @@ static char *fetch_url(const char *url)
 	curl_easy_setopt(curl, CURLOPT_USERAGENT, "ppastats/0.0");
 
 	if (curl_easy_perform(curl) == CURLE_OK) {
-
 		curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);
-		if (code == 200)
+
+		switch (code) {
+		case 200:
 			result = content->data;
+			break;
+		case 500:
+		case 502:
+		case 503:
+		case 504:
+			if (retries) {
+				fprintf(stderr, "Fetch failed: %ld\n", code);
+
+				if (debug)
+					printf("Wait 5s before retry.\n");
+				sleep(5);
+
+				free(content->data);
+				retries--;
+				goto retrieve;
+			}
+		default:
+			fprintf(stderr, "Fetch failed: %ld\n", code);
+		}
 	}
 
 	if (!result)
@@ -108,29 +147,51 @@ static json_object *get_json_object(const char *url)
 	return NULL;
 }
 
+#define json_object_to_bpph_list \
+json_object_to_binary_package_publishing_history_list
+
 struct binary_package_publishing_history * *
-get_binary_package_publishing_history_list(const char *archive_url)
+get_binary_package_publishing_history_list(const char *archive_url,
+					   const char *pkg_status)
 {
-	char *url = malloc(strlen(archive_url)+
-			   strlen(QUERY_GET_PUBLISHED_BINARIES)+
-			   1);
+	struct json_object *o_next;
+	char *url;
 	json_object *o;
-	struct binary_package_publishing_history **result;
+	void **result = NULL;
+
+	url = malloc(strlen(archive_url)+
+		     strlen(QUERY_GET_PUBLISHED_BINARIES)+
+		     (pkg_status ? strlen("&status=")+strlen(pkg_status) : 0)+
+		     1);
 
 	strcpy(url, archive_url);
 	strcat(url, QUERY_GET_PUBLISHED_BINARIES);
 
-	o = get_json_object(url);
-	free(url);
+	if (pkg_status) {
+		strcat(url, "&status=");
+		strcat(url, pkg_status);
+	}
 
-	if (!o)
-		return NULL;
+	while (url) {
+		o = get_json_object(url);
+		free(url);
+		url = NULL;
 
-	result = json_object_to_binary_package_publishing_history_list(o);
+		if (!o)
+			break;
 
-	json_object_put(o);
+		result = list_append_list(result,
+					  (void **)json_object_to_bpph_list(o));
 
-	return result;
+		o_next = json_object_object_get(o, "next_collection_link");
+
+		if (o_next)
+			url = strdup(json_object_get_string(o_next));
+
+		json_object_put(o);
+	}
+
+	return (struct binary_package_publishing_history **)result;
 }
 
 int get_download_count(const char *archive_url)
@@ -156,10 +217,14 @@ int get_download_count(const char *archive_url)
 	return result;
 }
 
-struct distro_arch_series *get_distro_arch_series(const char *url)
+const struct distro_arch_series *get_distro_arch_series(const char *url)
 {
 	json_object *obj;
-	struct distro_arch_series *distro;
+	const struct distro_arch_series *distro;
+
+	distro = cache_get(url);
+	if (distro)
+		return (struct distro_arch_series *)distro;
 
 	obj = get_json_object(url);
 
@@ -170,6 +235,31 @@ struct distro_arch_series *get_distro_arch_series(const char *url)
 
 	json_object_put(obj);
 
+	cache_put(url, distro, (void (*)(void *))&distro_arch_series_free);
+
+	return distro;
+}
+
+const struct distro_series *get_distro_series(const char *url)
+{
+	json_object *obj;
+	const struct distro_series *distro;
+
+	distro = cache_get(url);
+	if (distro)
+		return (struct distro_series *)distro;
+
+	obj = get_json_object(url);
+
+	if (!obj)
+		return NULL;
+
+	distro = json_object_to_distro_series(obj);
+
+	json_object_put(obj);
+
+	cache_put(url, distro, (void (*)(void *))&distro_series_free);
+
 	return distro;
 }
 
@@ -187,11 +277,21 @@ struct daily_download_total **get_daily_download_totals(const char *binary_url)
 
 	obj = get_json_object(url);
 
-	if (obj)
+	if (obj) {
 		result = json_object_to_daily_download_totals(obj);
+		json_object_put(obj);
+	}
 
 	free(url);
-	json_object_put(obj);
 
 	return result;
 }
+
+void lp_ws_cleanup()
+{
+	if (debug)
+		printf("DEBUG: cleanup CURL\n");
+
+	curl_easy_cleanup(curl);
+	curl_global_cleanup();
+}