group by 300 the result of http request getPublishedBinaries

[ppastats.git] / src / lp_ws.c

diff --git a/src/lp_ws.c b/src/lp_ws.c
index 5eca19a..f52f935 100644 (file)
--- a/src/lp_ws.c
+++ b/src/lp_ws.c
@@ -1,21 +1,24 @@
 /*
-    Copyright (C) 2011 jeanfi@gmail.com
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program; if not, write to the Free Software
-    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-    02110-1301 USA
-*/
+ * Copyright (C) 2011-2012 jeanfi@gmail.com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ */
+
+#include <libintl.h>
+#define _(String) gettext(String)
 
 #include <stdlib.h>
 #include <string.h>
@@ -26,16 +29,18 @@
 
 #include "cache.h"
 #include "list.h"
+#include "log.h"
 #include "lp_ws.h"
 #include "lp_json.h"
 #include "ppastats.h"
 
-static const char *QUERY_GET_PUBLISHED_BINARIES = "?ws.op=getPublishedBinaries";
+static const char *
+QUERY_GET_PUBLISHED_BINARIES = "?ws.op=getPublishedBinaries&ws.size=300";
 static const char *QUERY_GET_DOWNLOAD_COUNT = "?ws.op=getDownloadCount";
 static const char *
 QUERY_GET_DAILY_DOWNLOAD_TOTALS = "?ws.op=getDailyDownloadTotals";
 
-static const int DEFAULT_FETCH_RETRIES = 3;
+static const int DEFAULT_FETCH_RETRIES = 10;
 
 static CURL *curl;
 
@@ -58,25 +63,29 @@ static size_t cbk_curl(void *buffer, size_t size, size_t nmemb, void *userp)
        return realsize;
 }
 
-static char *fetch_url(const char *url)
+static void init()
 {
-       struct ucontent *content = malloc(sizeof(struct ucontent));
-       char *result;
-       long code;
-       int retries;
-
-       if (debug)
-               printf("DEBUG: fetch_url %s\n", url);
-
        if (!curl) {
-               if (debug)
-                       printf("DEBUG: initializing CURL\n");
+               log_debug(_("initializing CURL"));
                curl_global_init(CURL_GLOBAL_ALL);
                curl = curl_easy_init();
        }
 
        if (!curl)
                exit(EXIT_FAILURE);
+}
+
+static char *fetch_url(const char *url)
+{
+       struct ucontent *content = malloc(sizeof(struct ucontent));
+       char *result;
+       long code;
+       int retries;
+       unsigned int s;
+
+       log_debug(_("fetch_url(): %s"), url);
+
+       init();
 
        result = NULL;
 
@@ -104,23 +113,25 @@ static char *fetch_url(const char *url)
                case 502:
                case 503:
                case 504:
-                       if (retries) {
-                               fprintf(stderr,
-                                       "Fetch failed: with code %ld "
-                                       "for URL= %s\n",
-                                       code,
-                                       url);
+                       log_err(_("Fetch failed with code %ld for URL %s"),
+                               code,
+                               url);
 
-                               if (debug)
-                                       printf("Wait 5s before retry.\n");
-                               sleep(5);
+                       if (retries) {
+                               s = 2 * (DEFAULT_FETCH_RETRIES - retries) + 2;
+                               log_debug(_("Wait %ds before retry"), s);
+                               sleep(s);
 
                                free(content->data);
                                retries--;
                                goto retrieve;
                        }
+
+                       break;
                default:
-                       fprintf(stderr, "Fetch failed: %ld\n", code);
+                       log_err(_("Fetch failed with code %ld for URL %s"),
+                               code,
+                               url);
                }
        }
 
@@ -292,8 +303,7 @@ struct daily_download_total **get_daily_download_totals(const char *binary_url)
 
 void lp_ws_cleanup()
 {
-       if (debug)
-               printf("DEBUG: cleanup CURL\n");
+       log_debug(_("cleanup CURL"));
 
        curl_easy_cleanup(curl);
        curl_global_cleanup();