X-Git-Url: https://git.wpitchoune.net/gitweb/?a=blobdiff_plain;f=war%2Fsrc%2Fmain%2Fjava%2Fpnews%2Fservlet%2FArticleProvider.java;h=3bef20e712f1216025e89fbc98d22241328e5775;hb=180bf164e8751ae680b27be5302c681b955eba76;hp=b28bbf5701aea8765e427b7c02d7a8cfed8dd6d1;hpb=14d5e6993de84967141710b3d3c44edf055a71ec;p=pnews.git diff --git a/war/src/main/java/pnews/servlet/ArticleProvider.java b/war/src/main/java/pnews/servlet/ArticleProvider.java index b28bbf5..3bef20e 100644 --- a/war/src/main/java/pnews/servlet/ArticleProvider.java +++ b/war/src/main/java/pnews/servlet/ArticleProvider.java @@ -27,15 +27,17 @@ import com.rometools.rome.io.XmlReader; import pnews.Article; import pnews.Category; +import pnews.Feed; public class ArticleProvider { - public final static ArticleProvider singleton = new ArticleProvider(); private static final Logger LOG = Logger.getLogger(ArticleProvider.class.getName()); private final Map> articlesByCategory = new HashMap<>(); private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(2); + private final Config config; - private ArticleProvider() { - for (Category cat:Category.values()) + public ArticleProvider(Config config) { + this.config = config; + for (Category cat: config.getCategories()) scheduler.scheduleAtFixedRate(new Refresher(cat), 2, 600, TimeUnit.SECONDS); } @@ -43,61 +45,10 @@ public class ArticleProvider { XmlReader r; r = new XmlReader(new URL(u)); - XmlReader.setDefaultEncoding("UTF-8"); return new SyndFeedInput().build(r); } - private static Map getFeeds() { - Map result; - - result = new HashMap<>(); - - result.put(Category.TOP, - new String[] { - "http://www.francetvinfo.fr/titres.rss", - "http://www.rfi.fr/general/rss", - "http://www.cnews.fr/rss/une", - "http://www.ladepeche.fr/rss/a-la-une.rss", - "https://www.franceinter.fr/rss/a-la-une.xml", - "https://www.francebleu.fr/rss/a-la-une.xml", - "http://www.bfmtv.com/rss/info/flux-rss/flux-toutes-les-actualites/" - }); - - result.put(Category.SPORT, - new String[] { "http://www.france24.com/fr/sports/rss" }); - - result.put(Category.FRANCE, - new String[] { "http://www.france24.com/fr/france/rss", - "http://www.rfi.fr/france/rss"}); - - result.put(Category.EUROPE, - new String[] { "http://www.france24.com/fr/europe/rss" }); - - result.put(Category.MONDE, - new String[] { "http://www.france24.com/fr/actualites/rss" }); - - - result.put(Category.ECO, - new String[] { "http://www.france24.com/fr/economie/rss", - "http://www.rfi.fr/economie/rss" }); - - result.put(Category.ESSONNE, - new String[] { "http://www.tourisme-essonne.com/rss/actus/", - "http://www.ville-palaiseau.fr/rss/actualites.htm" }); - - result.put(Category.PEOPLE, - new String[] { "http://www.premiere.fr/rss/actu-live", - "http://www.purepeople.com/rss/news_t0.xml" - }); - - result.put(Category.TECHNOLOGIE, - new String[] { "http://feeds.feedburner.com/lesnumeriques/news", - "http://www.zdnet.fr/feeds/rss/actualites/"}); - - return result; - } - private List
getArticlesForUpdate(Category cat) { List
result; @@ -120,10 +71,15 @@ public class ArticleProvider { return false; } - private Article toArticle(String link, Category cat, SyndEntry entry, SyndFeed feed) { - String desc, title, thumbnail; + private static Article toArticle(String link, SyndEntry entry, SyndFeed feed) { + String desc, title, thumbnail, feedTitle, str; Date date; - + + feedTitle = feed.getTitle(); + if (feedTitle != null) { + feedTitle = feedTitle.trim(); + } + thumbnail = null; for (SyndEnclosure e: entry.getEnclosures()) { if (e.getType().startsWith("image/")) @@ -133,52 +89,45 @@ public class ArticleProvider { if (thumbnail == null && feed.getImage() != null) thumbnail = feed.getImage().getUrl(); - + + title = entry.getTitle().trim(); - if (entry.getDescription() != null) { - desc = Jsoup.parse(entry.getDescription().getValue()).text(); + if (entry.getDescription() != null) { + str = entry.getDescription().getValue(); + desc = Jsoup.parse(str).text(); } else { desc = null; - LOG.severe("No description for " + feed.getTitle() + " - " + title); + LOG.severe("No description for " + feedTitle + " - " + title); } date = entry.getPublishedDate(); if (date == null) date = entry.getUpdatedDate(); if (date == null) - LOG.severe("The article " + feed.getTitle() + " - " + title + " does not have a date"); + LOG.severe("The article " + feedTitle + " - " + title + " does not have a date"); - return new Article(link, cat, title, desc, thumbnail, date, title); + return new Article(link, title, desc, thumbnail, date, feedTitle); } - - private void addArticles(Category cat, SyndFeed feed) { - String thumbnail; - String desc, link, title, feedTitle, feedImage; - Date date; + String feedTitle; List
articles; Article a; feedTitle = feed.getTitle().trim(); - if (feed.getImage() != null) - feedImage = feed.getImage().getUrl(); - else - feedImage = null; - - LOG.info("addArticles " + cat.getId() + " " + feedTitle + " number of articles: " + feed.getEntries().size()); + LOG.info("addArticles " + cat.getLabel() + " " + feedTitle + " number of articles: " + feed.getEntries().size()); for (SyndEntry entry: feed.getEntries()) { - link = entry.getLink().trim(); + String link = entry.getLink().trim(); articles = getArticlesForUpdate(cat); if (exists(link, articles)) { LOG.fine("addArticles " + link + " is already present"); continue ; } - a = toArticle(link, cat, entry, feed); + a = ArticleStore.singleton.getArticle(link, ()->toArticle(link, entry, feed)); synchronized (articles) { articles.add(a); @@ -194,26 +143,25 @@ public class ArticleProvider { return -1; return o2.publicationDate.compareTo(o1.publicationDate); } - }); } } - LOG.info("addArticles done " + cat.getId()); + LOG.info("addArticles done " + cat.getLabel()); } - + private void retrieveArticles(Category cat) throws IllegalArgumentException, MalformedURLException, FeedException, IOException { - String[] feeds; + List feeds; - feeds = getFeeds().get(cat); + feeds = config.getFeedsByCategory().get(cat); if (feeds != null) - for (String str: feeds) + for (Feed f: feeds) try { - addArticles(cat, getSyndFeed(str)); + addArticles(cat, getSyndFeed(f.getURL())); } catch (Throwable e) { LOG.log(Level.SEVERE, - "retrieveArticles failure " + cat.getId() + " " + str, + "retrieveArticles failure " + cat.getLabel() + " " + f.toString(), e); } else @@ -244,28 +192,16 @@ public class ArticleProvider { } @Override - public void run() { - List
articles; - - LOG.info("refresher "+ category.getId()); + public void run() { + LOG.info("refresher "+ category.getLabel()); try { retrieveArticles(category); - - synchronized (articlesByCategory) { - articles = articlesByCategory.get(category); - if (articles != null && articles.size() > 100) { - articlesByCategory.put(category, - articles.subList(0, 100)); - - } - LOG.info("refresher " + category.getId() + " number of articles: " + articles.size()); - } } catch (IllegalArgumentException | FeedException | IOException e) { LOG.log(Level.SEVERE, "refresher failure", e); } - LOG.info("refresher "+ category.getId() + " done"); + LOG.info("refresher "+ category.getLabel() + " done"); } } }