X-Git-Url: http://git.wpitchoune.net/gitweb/?a=blobdiff_plain;f=war%2Fsrc%2Fmain%2Fjava%2Fpnews%2Fservlet%2FArticleProvider.java;h=ec741238956c04311e404d1f77bc92bc25b401c8;hb=6d94fd5a39e6f78a68201230bd57b9ceb95e125d;hp=d65002d30f83bfeeeaaa254caa94454b2ce7282d;hpb=5d0b7bff3d5c053abc0ac76c69d7cb1ae3978aba;p=pnews.git diff --git a/war/src/main/java/pnews/servlet/ArticleProvider.java b/war/src/main/java/pnews/servlet/ArticleProvider.java index d65002d..ec74123 100644 --- a/war/src/main/java/pnews/servlet/ArticleProvider.java +++ b/war/src/main/java/pnews/servlet/ArticleProvider.java @@ -27,15 +27,20 @@ import com.rometools.rome.io.XmlReader; import pnews.Article; import pnews.Category; +import pnews.EntityStat; +import pnews.Feed; +import pnews.NER; public class ArticleProvider { - public final static ArticleProvider singleton = new ArticleProvider(); - private static final Logger LOG = Logger.getLogger(ArticleProvider.class.getName()); + private static final String CLASS_NAME = ArticleProvider.class.getName(); + private static final Logger LOG = Logger.getLogger(CLASS_NAME); private final Map> articlesByCategory = new HashMap<>(); - private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(2); + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors()); + private final Config config; - private ArticleProvider() { - for (Category cat:Category.values()) + public ArticleProvider(Config config) { + this.config = config; + for (Category cat: config.getCategories()) scheduler.scheduleAtFixedRate(new Refresher(cat), 2, 600, TimeUnit.SECONDS); } @@ -47,57 +52,6 @@ public class ArticleProvider { return new SyndFeedInput().build(r); } - private static Map getFeeds() { - Map result; - - result = new HashMap<>(); - - result.put(Category.TOP, - new String[] { - "http://www.francetvinfo.fr/titres.rss", - "http://www.rfi.fr/general/rss", - "http://www.cnews.fr/rss/une", - "http://www.ladepeche.fr/rss/a-la-une.rss", - "https://www.franceinter.fr/rss/a-la-une.xml", - "https://www.francebleu.fr/rss/a-la-une.xml", - "http://www.bfmtv.com/rss/info/flux-rss/flux-toutes-les-actualites/" - }); - - result.put(Category.SPORT, - new String[] { "http://www.france24.com/fr/sports/rss" }); - - result.put(Category.FRANCE, - new String[] { "http://www.france24.com/fr/france/rss", - "http://www.rfi.fr/france/rss"}); - - result.put(Category.EUROPE, - new String[] { "http://www.france24.com/fr/europe/rss" }); - - result.put(Category.MONDE, - new String[] { "http://www.france24.com/fr/actualites/rss" }); - - - result.put(Category.ECO, - new String[] { "http://www.france24.com/fr/economie/rss", - "http://www.rfi.fr/economie/rss" }); - - result.put(Category.ESSONNE, - new String[] { "http://www.tourisme-essonne.com/rss/actus/", - "http://www.ville-palaiseau.fr/rss/actualites.htm" }); - - result.put(Category.PEOPLE, - new String[] { "http://www.premiere.fr/rss/actu-live", - "http://www.purepeople.com/rss/news_t0.xml" - }); - - result.put(Category.TECHNOLOGIE, - new String[] { "http://feeds.feedburner.com/lesnumeriques/news", - "http://www.zdnet.fr/feeds/rss/actualites/", - "https://www-03.ibm.com/press/fr/fr/rssfeed.wss?keyword=null&maxFeed=&feedType=RSS&topic=all"}); - - return result; - } - private List
getArticlesForUpdate(Category cat) { List
result; @@ -120,9 +74,10 @@ public class ArticleProvider { return false; } - private static Article toArticle(String link, SyndEntry entry, SyndFeed feed) { + private Article toArticle(String link, SyndEntry entry, SyndFeed feed, String lang) { String desc, title, thumbnail, feedTitle, str; Date date; + List entities; feedTitle = feed.getTitle(); if (feedTitle != null) { @@ -135,11 +90,7 @@ public class ArticleProvider { thumbnail = e.getUrl(); break; } - - if (thumbnail == null && feed.getImage() != null) - thumbnail = feed.getImage().getUrl(); - - + title = entry.getTitle().trim(); if (entry.getDescription() != null) { @@ -156,7 +107,17 @@ public class ArticleProvider { if (date == null) LOG.severe("The article " + feedTitle + " - " + title + " does not have a date"); - return new Article(link, title, desc, thumbnail, date, feedTitle); + + entities = new ArrayList<>(); + if (desc != null && lang.equals("en")) + try { + NER.classify(title, entities, config); + NER.classify(desc, entities, config); + } catch (ClassCastException | ClassNotFoundException | IOException e1) { + LOG.log(Level.SEVERE, "Cannot classify " + feedTitle, e1); + } + + return new Article(link, title, desc, thumbnail, date, feedTitle, entities.toArray(new String[0])); } private void addArticles(Category cat, SyndFeed feed) { @@ -166,7 +127,7 @@ public class ArticleProvider { feedTitle = feed.getTitle().trim(); - LOG.info("addArticles " + cat.getId() + " " + feedTitle + " number of articles: " + feed.getEntries().size()); + LOG.info("addArticles " + cat.getLabel() + " " + feedTitle + " number of articles: " + feed.getEntries().size()); for (SyndEntry entry: feed.getEntries()) { String link = entry.getLink().trim(); @@ -176,7 +137,7 @@ public class ArticleProvider { continue ; } - a = ArticleStore.singleton.getArticle(link, ()->toArticle(link, entry, feed)); + a = ArticleStore.singleton.getArticle(link, ()->toArticle(link, entry, feed, cat.getLanguage())); synchronized (articles) { articles.add(a); @@ -196,21 +157,21 @@ public class ArticleProvider { } } - LOG.info("addArticles done " + cat.getId()); + LOG.info("addArticles done " + cat.getLabel()); } private void retrieveArticles(Category cat) throws IllegalArgumentException, MalformedURLException, FeedException, IOException { - String[] feeds; + List feeds; - feeds = getFeeds().get(cat); + feeds = config.getFeedsByCategory().get(cat); if (feeds != null) - for (String str: feeds) + for (Feed f: feeds) try { - addArticles(cat, getSyndFeed(str)); + addArticles(cat, getSyndFeed(f.getURL())); } catch (Throwable e) { LOG.log(Level.SEVERE, - "retrieveArticles failure " + cat.getId() + " " + str, + "retrieveArticles failure " + cat.getLabel() + " " + f.toString(), e); } else @@ -220,19 +181,66 @@ public class ArticleProvider { /** * Returns a copy. */ - public List
getArticles(Category cat) + public List
getArticles(Category cat, String entity) throws IllegalArgumentException, MalformedURLException, FeedException, IOException { - List
articles; + List
articles, result; synchronized (articlesByCategory) { articles = getArticlesForUpdate(cat); } - synchronized (articles) { - return new ArrayList<>(articles); + synchronized (articles) { + if (entity == null) + return new ArrayList<>(articles); + + result = new ArrayList<>(articles.size()); + for (Article a: articles) + if (a.hasEntity(entity)) + result.add(a); + + return result; } } + public List getEntityStats(Category cat) throws IllegalArgumentException, MalformedURLException, FeedException, IOException { + List
articles; + Map entities; + final String FUNCTION_NAME = "getEntities"; + EntityStat s; + List stats; + + LOG.entering(CLASS_NAME, FUNCTION_NAME, cat); + + articles = getArticles(cat, null); + + entities = new HashMap<>(); + for (Article a: articles) + if (a.getEntities() != null) { + for (String e: a.getEntities()) { + s = entities.get(e); + if (s == null) { + s = new EntityStat(e); + entities.put(e, s); + } + s.increment(); + } + } + + stats = new ArrayList<>(entities.values()); + stats.sort(new Comparator() { + + @Override + public int compare(EntityStat o1, EntityStat o2) { + return Integer.compare(o2.getCount(), o1.getCount()); + } + + }); + + LOG.exiting(CLASS_NAME, FUNCTION_NAME, stats); + + return stats; + } + private class Refresher implements Runnable { private final Category category; @@ -242,7 +250,7 @@ public class ArticleProvider { @Override public void run() { - LOG.info("refresher "+ category.getId()); + LOG.info("refresher "+ category.getLabel()); try { retrieveArticles(category); @@ -250,7 +258,7 @@ public class ArticleProvider { LOG.log(Level.SEVERE, "refresher failure", e); } - LOG.info("refresher "+ category.getId() + " done"); + LOG.info("refresher "+ category.getLabel() + " done"); } } }