X-Git-Url: http://git.wpitchoune.net/gitweb/?a=blobdiff_plain;f=src%2Flp_ws.c;h=fcf08c41b3ce736e1a66510b510bada33df04b8f;hb=cbe5f873c5f0306a7a5d2f431992dbbb6edbe332;hp=a6b95d11bb1f346135df750e851fb2304e97fc99;hpb=5ed73c995d6fcb5f3b4626bf7ebf609fc52f5d94;p=ppastats.git diff --git a/src/lp_ws.c b/src/lp_ws.c index a6b95d1..fcf08c4 100644 --- a/src/lp_ws.c +++ b/src/lp_ws.c @@ -1,134 +1,186 @@ /* - Copyright (C) 2011 jeanfi@gmail.com - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - 02110-1301 USA -*/ - + * Copyright (C) 2011-2012 jeanfi@gmail.com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA + */ + +#include +#define _(String) gettext(String) + +#include #include #include -#include #include +#include "cache.h" +#include "fcache.h" +#include "http.h" +#include "list.h" +#include "log.h" #include "lp_ws.h" #include "lp_json.h" +#include "ppastats.h" -#define QUERY_GET_PUBLISHED_BINARIES \ - "?ws.op=getPublishedBinaries&status=Published" -#define QUERY_GET_DOWNLOAD_COUNT "?ws.op=getDownloadCount" -#define QUERY_GET_DAILY_DOWNLOAD_TOTALS \ - "?ws.op=getDailyDownloadTotals" +static const char * +QUERY_GET_PUBLISHED_BINARIES = "?ws.op=getPublishedBinaries&ws.size=150"; +static const char *QUERY_GET_DOWNLOAD_COUNT = "?ws.op=getDownloadCount"; +static const char * +QUERY_GET_DAILY_DOWNLOAD_TOTALS = "?ws.op=getDailyDownloadTotals"; -static CURL *curl; +static json_object *get_json_object(const char *url) +{ + json_object *obj = NULL; + char *body; -struct ucontent { - char *data; - size_t len; -}; + body = get_url_content(url, 0); -static size_t cbk_curl(void *buffer, size_t size, size_t nmemb, void *userp) -{ - size_t realsize = size * nmemb; - struct ucontent *mem = (struct ucontent *)userp; + if (body) { + obj = json_tokener_parse(body); - mem->data = realloc(mem->data, mem->len + realsize + 1); + free(body); - memcpy(&(mem->data[mem->len]), buffer, realsize); - mem->len += realsize; - mem->data[mem->len] = 0; + return obj; + } - return realsize; + return NULL; } -static char *fetch_url(const char *url) +char *get_bpph_list_cache_key(const char *archive_url) { - struct ucontent *content = malloc(sizeof(struct ucontent)); - char *result = NULL; - long code; - - if (!curl) - curl = curl_easy_init(); + char *key; - if (!curl) - exit(EXIT_FAILURE); + key = malloc(strlen(archive_url + 7) + strlen("/bpph") + 1); + sprintf(key, "%s/bpph", archive_url + 7); - content->data = malloc(1); - content->data[0] = '\0'; - content->len = 0; + return key; +} - curl_easy_setopt(curl, CURLOPT_URL, url); - curl_easy_setopt(curl, CURLOPT_VERBOSE, 0); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cbk_curl); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, content); - curl_easy_setopt(curl, CURLOPT_USERAGENT, "ppastats/0.0"); +struct bpph **get_bpph_list_from_cache(const char *key) +{ + char *content; + struct bpph **list; + json_object *json; - if (curl_easy_perform(curl) == CURLE_OK) { + content = fcache_get(key); + if (!content) + return NULL; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); - if (code == 200) - result = content->data; - } + json = json_tokener_parse(content); + if (!json) + return NULL; - if (!result) - free(content->data); + list = json_object_to_bpph_list(json); + json_object_put(json); free(content); - return result; + return list; } -static json_object *get_json_object(const char *url) +struct tm *get_last_creation_date(struct bpph **list) { - json_object *obj = NULL; - char *body; + time_t last, t; + struct bpph **cur; - body = fetch_url(url); + last = 0; - if (body) { - obj = json_tokener_parse(body); - - free(body); - - return obj; - } + if (list) + for (cur = list; *cur; cur++) { + t = mktime(&(*cur)->date_created); + if (t > last) + last = t; + } - return NULL; + return localtime(&last); } -struct binary_package_publishing_history * * -get_binary_package_publishing_history_list(const char *archive_url) +struct bpph **get_bpph_list(const char *archive_url, const char *pkg_status) { - char *url = malloc(strlen(archive_url)+ - strlen(QUERY_GET_PUBLISHED_BINARIES)+ - 1); - json_object *o; - struct binary_package_publishing_history **result; - + char *url, *key, *tmp; + struct bpph **result = NULL; + struct json_object *o, *bpph_json, *o_next; + char *created_since_date; + struct tm *tm; + int ok; + + url = malloc(strlen(archive_url) + + strlen(QUERY_GET_PUBLISHED_BINARIES) + + 1); strcpy(url, archive_url); strcat(url, QUERY_GET_PUBLISHED_BINARIES); - o = get_json_object(url); - free(url); + key = get_bpph_list_cache_key(archive_url); - if (!o) - return NULL; + result = get_bpph_list_from_cache(key); + + if (result) { + tm = get_last_creation_date(result); - result = json_object_to_binary_package_publishing_history_list(o); + created_since_date = malloc(200); + strftime(created_since_date, + 100, + "%FT%T", + tm); + + printf("Update package since: %s\n", created_since_date); + + tmp = malloc(strlen(url) + + strlen("&created_since_date=") + + strlen(created_since_date)+1); + strcpy(tmp, url); + strcat(tmp, "&created_since_date="); + strcat(tmp, created_since_date); + + free(url); + url = tmp; + + free(created_since_date); + } - json_object_put(o); + ok = 1; + while (url) { + o = get_json_object(url); + free(url); + url = NULL; + + if (!o) { + ok = 0; + break; + } + + result = bpph_list_append_list(result, + json_object_to_bpph_list(o)); + + o_next = json_object_object_get(o, "next_collection_link"); + + if (o_next) + url = strdup(json_object_get_string(o_next)); + + json_object_put(o); + + } + + if (ok) { + bpph_json = bpph_list_to_json(result); + fcache_put(key, json_object_to_json_string(bpph_json)); + json_object_put(bpph_json); + } + + free(key); return result; } @@ -156,12 +208,24 @@ int get_download_count(const char *archive_url) return result; } -struct distro_arch_series *get_distro_arch_series(const char *url) +const struct distro_arch_series *get_distro_arch_series(const char *url) { json_object *obj; - struct distro_arch_series *distro; + const struct distro_arch_series *distro; + char *content; - obj = get_json_object(url); + distro = cache_get(url); + if (distro) + return (struct distro_arch_series *)distro; + + content = get_url_content(url, 1); + + if (!content) + return NULL; + + obj = json_tokener_parse(content); + + free(content); if (!obj) return NULL; @@ -170,6 +234,39 @@ struct distro_arch_series *get_distro_arch_series(const char *url) json_object_put(obj); + cache_put(url, distro, (void (*)(void *))&distro_arch_series_free); + + return distro; +} + +const struct distro_series *get_distro_series(const char *url) +{ + json_object *obj; + const struct distro_series *distro; + char *content; + + distro = cache_get(url); + if (distro) + return (struct distro_series *)distro; + + content = get_url_content(url, 1); + + if (!content) + return NULL; + + obj = json_tokener_parse(content); + + free(content); + + if (!obj) + return NULL; + + distro = json_object_to_distro_series(obj); + + json_object_put(obj); + + cache_put(url, distro, (void (*)(void *))&distro_series_free); + return distro; } @@ -187,11 +284,13 @@ struct daily_download_total **get_daily_download_totals(const char *binary_url) obj = get_json_object(url); - if (obj) + if (obj) { result = json_object_to_daily_download_totals(obj); + json_object_put(obj); + } free(url); - json_object_put(obj); return result; } +