use file cache for distros, refactoring http fcts

[ppastats.git] / src / lp_ws.c

diff --git a/src/lp_ws.c b/src/lp_ws.c
index 023ba9b..63a8d36 100644 (file)
--- a/src/lp_ws.c
+++ b/src/lp_ws.c
@@ -22,13 +22,12 @@
 
 #include <stdlib.h>
 #include <string.h>
-#include <unistd.h>
 
-#include <curl/curl.h>
 #include <json/json.h>
 
 #include "cache.h"
 #include "fcache.h"
+#include "http.h"
 #include "list.h"
 #include "log.h"
 #include "lp_ws.h"
@@ -41,115 +40,12 @@ static const char *QUERY_GET_DOWNLOAD_COUNT = "?ws.op=getDownloadCount";
 static const char *
 QUERY_GET_DAILY_DOWNLOAD_TOTALS = "?ws.op=getDailyDownloadTotals";
 
-static const int DEFAULT_FETCH_RETRIES = 10;
-
-static CURL *curl;
-
-struct ucontent {
-       char *data;
-       size_t len;
-};
-
-static size_t cbk_curl(void *buffer, size_t size, size_t nmemb, void *userp)
-{
-       size_t realsize = size * nmemb;
-       struct ucontent *mem = (struct ucontent *)userp;
-
-       mem->data = realloc(mem->data, mem->len + realsize + 1);
-
-       memcpy(&(mem->data[mem->len]), buffer, realsize);
-       mem->len += realsize;
-       mem->data[mem->len] = 0;
-
-       return realsize;
-}
-
-static void init()
-{
-       if (!curl) {
-               log_debug(_("initializing CURL"));
-               curl_global_init(CURL_GLOBAL_ALL);
-               curl = curl_easy_init();
-       }
-
-       if (!curl)
-               exit(EXIT_FAILURE);
-}
-
-static char *fetch_url(const char *url)
-{
-       struct ucontent *content = malloc(sizeof(struct ucontent));
-       char *result;
-       long code;
-       int retries;
-       unsigned int s;
-
-       log_debug(_("fetch_url(): %s"), url);
-
-       init();
-
-       result = NULL;
-
-       retries = DEFAULT_FETCH_RETRIES;
-
- retrieve:
-       content->data = malloc(1);
-       content->data[0] = '\0';
-       content->len = 0;
-
-       curl_easy_setopt(curl, CURLOPT_URL, url);
-       curl_easy_setopt(curl, CURLOPT_VERBOSE, 0);
-       curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cbk_curl);
-       curl_easy_setopt(curl, CURLOPT_WRITEDATA, content);
-       curl_easy_setopt(curl, CURLOPT_USERAGENT, "ppastats/0.0");
-
-       if (curl_easy_perform(curl) == CURLE_OK) {
-               curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);
-
-               switch (code) {
-               case 200:
-                       result = content->data;
-                       break;
-               case 500:
-               case 502:
-               case 503:
-               case 504:
-                       log_err(_("Fetch failed with code %ld for URL %s"),
-                               code,
-                               url);
-
-                       if (retries) {
-                               s = 2 * (DEFAULT_FETCH_RETRIES - retries) + 2;
-                               log_debug(_("Wait %ds before retry"), s);
-                               sleep(s);
-
-                               free(content->data);
-                               retries--;
-                               goto retrieve;
-                       }
-
-                       break;
-               default:
-                       log_err(_("Fetch failed with code %ld for URL %s"),
-                               code,
-                               url);
-               }
-       }
-
-       if (!result)
-               free(content->data);
-
-       free(content);
-
-       return result;
-}
-
 static json_object *get_json_object(const char *url)
 {
        json_object *obj = NULL;
        char *body;
 
-       body = fetch_url(url);
+       body = get_url_content(url, 0);
 
        if (body) {
                obj = json_tokener_parse(body);
@@ -242,14 +138,10 @@ const struct distro_arch_series *get_distro_arch_series(const char *url)
        if (distro)
                return (struct distro_arch_series *)distro;
 
-       content = fcache_get(url + 7);
-       if (!content) {
-               content = fetch_url(url);
-               if (content)
-                       fcache_put(url + 7, content);
-               else
-                       return NULL;
-       }
+       content = get_url_content(url, 1);
+       
+       if (!content)
+               return NULL;
 
        obj = json_tokener_parse(content);
 
@@ -271,12 +163,20 @@ const struct distro_series *get_distro_series(const char *url)
 {
        json_object *obj;
        const struct distro_series *distro;
+       char *content;
 
        distro = cache_get(url);
        if (distro)
                return (struct distro_series *)distro;
 
-       obj = get_json_object(url);
+       content = get_url_content(url, 1);
+
+       if (!content)
+               return NULL;
+
+       obj = json_tokener_parse(content);
+
+       free(content);
 
        if (!obj)
                return NULL;
@@ -314,10 +214,3 @@ struct daily_download_total **get_daily_download_totals(const char *binary_url)
        return result;
 }
 
-void lp_ws_cleanup()
-{
-       log_debug(_("cleanup CURL"));
-
-       curl_easy_cleanup(curl);
-       curl_global_cleanup();
-}