src/lp_ws.c

   1 /*
   2  * Copyright (C) 2011-2014 jeanfi@gmail.com
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License as
   6  * published by the Free Software Foundation; either version 2 of the
   7  * License, or (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful, but
  10  * WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  17  * 02110-1301 USA
  18  */
  19
  20 #include <libintl.h>
  21 #define _(String) gettext(String)
  22
  23 #include <stdio.h>
  24 #include <stdlib.h>
  25 #include <string.h>
  26 #include <sys/time.h>
  27 #include <time.h>
  28
  29 #include <json.h>
  30
  31 #include <cache.h>
  32 #include <fcache.h>
  33 #include <http.h>
  34 #include <list.h>
  35 #include <lp_ws.h>
  36 #include <lp_json.h>
  37 #include <plog.h>
  38 #include <ppastats.h>
  39 #include <ptime.h>
  40
  41 /** Default ws.size value for the getPublishedBinaries request. */
  42 static const int DEFAULT_WS_SIZE = 150;
  43
  44 static const char *QUERY_GET_DOWNLOAD_COUNT = "?ws.op=getDownloadCount";
  45 static const char *
  46 QUERY_GET_DAILY_DOWNLOAD_TOTALS = "?ws.op=getDailyDownloadTotals";
  47
  48 static json_object *get_json_object(const char *url)
  49 {
  50         json_object *obj = NULL;
  51         char *body;
  52
  53         body = get_url_content(url, 0);
  54
  55         if (body) {
  56                 obj = json_tokener_parse(body);
  57
  58                 free(body);
  59
  60                 return obj;
  61         }
  62
  63         return NULL;
  64 }
  65
  66 static char *get_bpph_list_cache_key(const char *archive_url)
  67 {
  68         char *key;
  69
  70         key = malloc(strlen(archive_url + 7) + strlen("/bpph") + 1);
  71         sprintf(key, "%s/bpph", archive_url + 7);
  72
  73         return key;
  74 }
  75
  76 static char *get_ddts_list_cache_key(const char *url)
  77 {
  78         char *key;
  79
  80         key = malloc(strlen(url + 7) + strlen("/ddts") + 1);
  81         sprintf(key, "%s/ddts", url + 7);
  82
  83         return key;
  84 }
  85
  86 static struct bpph **get_bpph_list_from_cache(const char *key)
  87 {
  88         char *content;
  89         struct bpph **list;
  90         json_object *json;
  91
  92         content = fcache_get(key);
  93         if (!content)
  94                 return NULL;
  95
  96         json = json_tokener_parse(content);
  97         if (!json)
  98                 return NULL;
  99
 100         list = json_object_to_bpph_list(json);
 101
 102         json_object_put(json);
 103         free(content);
 104
 105         return list;
 106 }
 107
 108 static char *get_last_creation_date(struct bpph **list)
 109 {
 110         time_t last, t;
 111         struct bpph **cur;
 112
 113         last = 0;
 114
 115         if (list)
 116                 for (cur = list; *cur; cur++) {
 117                         t = (*cur)->date_created;
 118                         if (t > last)
 119                                 last = t;
 120                 }
 121
 122         if (last)
 123                 return time_to_ISO8601_time(&last);
 124         else
 125                 return NULL;
 126 }
 127
 128 /*
 129  * 'archive_url': LP URL of the archive.
 130  * 'size': size of the reply array. Between 1-300, else default value is used.
 131  */
 132 static char *create_query_get_bpph(const char *archive_url,
 133                                    const char *status,
 134                                    int size)
 135 {
 136         static const char *default_opt = "?ws.op=getPublishedBinaries&ws.size=";
 137         static const char *status_opt = "&status=";
 138         char *url;
 139         size_t n;
 140
 141         if (size < 1 || size > 300)
 142                 size = DEFAULT_WS_SIZE;
 143
 144         n = strlen(archive_url) + strlen(default_opt) + 3 + 1;
 145
 146         if (status)
 147                 n += strlen(status_opt) + strlen(status);
 148
 149         url = malloc(n);
 150         sprintf(url, "%s%s%d", archive_url, default_opt, size);
 151
 152         if (status) {
 153                 strcat(url, status_opt);
 154                 strcat(url, status);
 155         }
 156
 157         return url;
 158 }
 159
 160 struct bpph **get_bpph_list(const char *archive_url,
 161                             const char *pkg_status,
 162                             int ws_size)
 163 {
 164         char *url, *key, *tmp;
 165         struct bpph **result;
 166         struct json_object *o, *bpph_json, *o_next;
 167         char *date;
 168         int ok;
 169
 170         url = create_query_get_bpph(archive_url, pkg_status, ws_size);
 171
 172         key = get_bpph_list_cache_key(archive_url);
 173
 174         result = get_bpph_list_from_cache(key);
 175
 176         if (result) {
 177                 date = get_last_creation_date(result);
 178
 179                 if (date) {
 180                         tmp = malloc(strlen(url)
 181                                      + strlen("&created_since_date=")
 182                                      + strlen(date)+1);
 183                         strcpy(tmp, url);
 184                         strcat(tmp, "&created_since_date=");
 185                         strcat(tmp, date);
 186
 187                         free(url);
 188                         url = tmp;
 189
 190                         free(date);
 191                 }
 192         }
 193
 194         ok = 1;
 195         while (url) {
 196                 o = get_json_object(url);
 197                 free(url);
 198                 url = NULL;
 199
 200                 if (!o) {
 201                         ok = 0;
 202                         break;
 203                 }
 204
 205                 result = bpph_list_append_list(result,
 206                                                json_object_to_bpph_list(o));
 207
 208                 o_next = json_object_object_get(o, "next_collection_link");
 209
 210                 if (o_next)
 211                         url = strdup(json_object_get_string(o_next));
 212
 213                 json_object_put(o);
 214         }
 215
 216         if (ok) {
 217                 bpph_json = bpph_list_to_json(result);
 218                 fcache_put(key, json_object_to_json_string(bpph_json));
 219                 json_object_put(bpph_json);
 220         }
 221
 222         free(key);
 223
 224         return result;
 225 }
 226
 227 int get_download_count(const char *archive_url)
 228 {
 229         int n = strlen(archive_url) + strlen(QUERY_GET_DOWNLOAD_COUNT) + 1;
 230         char *url = malloc(n);
 231         int result;
 232         json_object *obj;
 233
 234         strcpy(url, archive_url);
 235         strcat(url, QUERY_GET_DOWNLOAD_COUNT);
 236
 237         obj = get_json_object(url);
 238         free(url);
 239
 240         if (!obj)
 241                 return -1;
 242
 243         result = json_object_get_int(obj);
 244
 245         json_object_put(obj);
 246
 247         return result;
 248 }
 249
 250 const struct distro_arch_series *get_distro_arch_series(const char *url)
 251 {
 252         json_object *obj;
 253         const struct distro_arch_series *distro;
 254         char *content;
 255
 256         distro = cache_get(url);
 257         if (distro)
 258                 return (struct distro_arch_series *)distro;
 259
 260         content = get_url_content(url, 1);
 261
 262         if (!content)
 263                 return NULL;
 264
 265         obj = json_tokener_parse(content);
 266
 267         free(content);
 268
 269         if (!obj)
 270                 return NULL;
 271
 272         distro = json_object_to_distro_arch_series(obj);
 273
 274         json_object_put(obj);
 275
 276         cache_put(url, distro, (void (*)(void *))&distro_arch_series_free);
 277
 278         return distro;
 279 }
 280
 281 const struct distro_series *get_distro_series(const char *url)
 282 {
 283         json_object *obj;
 284         const struct distro_series *distro;
 285         char *content;
 286
 287         distro = cache_get(url);
 288         if (distro)
 289                 return (struct distro_series *)distro;
 290
 291         content = get_url_content(url, 1);
 292
 293         if (!content)
 294                 return NULL;
 295
 296         obj = json_tokener_parse(content);
 297
 298         free(content);
 299
 300         if (!obj)
 301                 return NULL;
 302
 303         distro = json_object_to_distro_series(obj);
 304
 305         json_object_put(obj);
 306
 307         cache_put(url, distro, (void (*)(void *))&distro_series_free);
 308
 309         return distro;
 310 }
 311
 312 /*
 313   Convert ddts older than 4 weeks to the same JSON representation than
 314   the LP one.  Newer ddts are not stored in the cache because the data
 315   may change during following days. It avoids to miss downloads which
 316   are not yet taken in consideration by LP.
 317  */
 318 static json_object *ddts_to_json_for_cache(struct daily_download_total **ddts)
 319 {
 320         json_object *j_ddts;
 321         struct daily_download_total *ddt;
 322         char *date;
 323         struct timeval *tv;
 324         time_t t;
 325         double d;
 326
 327         j_ddts = json_object_new_object();
 328
 329         tv = malloc(sizeof(struct timeval));
 330         gettimeofday(tv, NULL);
 331
 332         while (ddts && *ddts) {
 333                 ddt = *ddts;
 334
 335                 t = mktime(&(ddt->date));
 336
 337                 d = difftime(tv->tv_sec, t);
 338
 339                 if (d > 4 * 7 * 24 * 60 * 60) { /* older than 4 weeks */
 340                         date = tm_to_ISO8601_date(&ddt->date);
 341                         json_object_object_add(j_ddts,
 342                                                date,
 343                                                json_object_new_int(ddt->count));
 344                         free(date);
 345                 }
 346
 347                 ddts++;
 348         }
 349
 350         free(tv);
 351
 352         return j_ddts;
 353 }
 354
 355 char *create_ddts_query(const char *binary_url, time_t st, time_t et)
 356 {
 357         char *q;
 358         char *sdate, *edate;
 359
 360         if (st) {
 361                 sdate = time_to_ISO8601_date(&st);
 362
 363                 q = malloc(strlen(binary_url)
 364                            + strlen(QUERY_GET_DAILY_DOWNLOAD_TOTALS)
 365                            + strlen("&start_date=YYYY-MM-DD")
 366                            + strlen("&end_date=YYYY-MM-DD")
 367                            + 1);
 368                 strcpy(q, binary_url);
 369                 strcat(q, QUERY_GET_DAILY_DOWNLOAD_TOTALS);
 370                 strcat(q, "&start_date=");
 371                 strcat(q, sdate);
 372
 373                 if (et > 0) {
 374                         edate = time_to_ISO8601_date(&et);
 375                         strcat(q, "&end_date=");
 376                         strcat(q, edate);
 377                         free(edate);
 378                 }
 379
 380                 free(sdate);
 381         } else {
 382                 q = malloc(strlen(binary_url)
 383                            + strlen(QUERY_GET_DAILY_DOWNLOAD_TOTALS)
 384                            + 1);
 385                 strcpy(q, binary_url);
 386                 strcat(q, QUERY_GET_DAILY_DOWNLOAD_TOTALS);
 387         }
 388
 389         return q;
 390 }
 391
 392 static struct daily_download_total **retrieve_ddts(const char *binary_url,
 393                                                    time_t date_since)
 394 {
 395         char *url;
 396         json_object *json;
 397         struct daily_download_total **ddts, **tmp;
 398         time_t crt;
 399
 400         url = create_ddts_query(binary_url, date_since, 0);
 401         json = get_json_object(url);
 402         free(url);
 403
 404         if (json) {
 405                 ddts = json_object_to_daily_download_totals(json);
 406                 json_object_put(json);
 407         } else {
 408                 crt = time(NULL);
 409                 ddts = NULL;
 410
 411                 while (date_since < crt) {
 412                         url = create_ddts_query(binary_url,
 413                                                 date_since,
 414                                                 date_since);
 415                         json = get_json_object(url);
 416                         free(url);
 417
 418                         if (!json)
 419                                 break;
 420
 421                         tmp = json_object_to_daily_download_totals(json);
 422                         json_object_put(json);
 423                         ddts = ddts_merge(ddts, tmp);
 424                         free(tmp);
 425
 426                         date_since = date_since + 24 * 60 * 60; /* +1 day */
 427
 428                         url = create_ddts_query(binary_url, date_since, 0);
 429                         json = get_json_object(url);
 430                         free(url);
 431
 432                         if (json) {
 433                                 tmp = json_object_to_daily_download_totals
 434                                         (json);
 435                                 json_object_put(json);
 436                                 ddts = ddts_merge(ddts, tmp);
 437                                 free(tmp);
 438                                 break;
 439                         }
 440                 }
 441         }
 442
 443         return ddts;
 444 }
 445
 446 struct daily_download_total **get_daily_download_totals(const char *binary_url,
 447                                                         time_t date_created)
 448 {
 449         char *key, *content;
 450         json_object *j_ddts, *json;
 451         struct daily_download_total **retrieved_ddts = NULL;
 452         struct daily_download_total **cached_ddts;
 453         struct daily_download_total **ddts;
 454         time_t last_t;
 455
 456         key = get_ddts_list_cache_key(binary_url);
 457
 458         content = fcache_get(key);
 459         if (content) {
 460                 json = json_tokener_parse(content);
 461                 free(content);
 462         } else {
 463                 json = NULL;
 464         }
 465
 466         if (json) {
 467                 cached_ddts = json_object_to_daily_download_totals(json);
 468                 json_object_put(json);
 469                 last_t = ddts_get_last_date(cached_ddts);
 470         } else {
 471                 last_t = 0;
 472                 cached_ddts = NULL;
 473         }
 474
 475         if (last_t > 0)
 476                 retrieved_ddts = retrieve_ddts(binary_url, last_t);
 477         else
 478                 retrieved_ddts = retrieve_ddts(binary_url, date_created);
 479
 480         ddts = ddts_merge(cached_ddts, retrieved_ddts);
 481
 482         if (ddts) {
 483                 j_ddts = ddts_to_json_for_cache(ddts);
 484                 fcache_put(key, json_object_get_string(j_ddts));
 485                 json_object_put(j_ddts);
 486         }
 487         free(key);
 488
 489         if (ddts != cached_ddts)
 490                 daily_download_total_list_free(cached_ddts);
 491         daily_download_total_list_free(retrieved_ddts);
 492
 493         return ddts;
 494 }
 495