fixed concurent exception when the list of articles is updated and requested

[pnews.git] / war / src / main / java / pnews / servlet / ArticleProvider.java

diff --git a/war/src/main/java/pnews/servlet/ArticleProvider.java b/war/src/main/java/pnews/servlet/ArticleProvider.java
index 010317a..55e098a 100644 (file)
--- a/war/src/main/java/pnews/servlet/ArticleProvider.java
+++ b/war/src/main/java/pnews/servlet/ArticleProvider.java
@@ -1,7 +1,6 @@
 package pnews.servlet;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
@@ -18,13 +17,13 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.jsoup.Jsoup;
-import org.xml.sax.InputSource;
 
 import com.rometools.rome.feed.synd.SyndEnclosure;
 import com.rometools.rome.feed.synd.SyndEntry;
 import com.rometools.rome.feed.synd.SyndFeed;
 import com.rometools.rome.io.FeedException;
 import com.rometools.rome.io.SyndFeedInput;
+import com.rometools.rome.io.XmlReader;
 
 import pnews.Article;
 import pnews.Category;
@@ -37,15 +36,16 @@ public class ArticleProvider {
         
         private ArticleProvider() {      
                 for (Category cat:Category.values())
-                        scheduler.scheduleAtFixedRate(new Refresher(cat), 2, 120, TimeUnit.SECONDS);
+                        scheduler.scheduleAtFixedRate(new Refresher(cat), 2, 600, TimeUnit.SECONDS);
         }
         
         private static SyndFeed getSyndFeed(String u) throws IllegalArgumentException, FeedException, MalformedURLException, IOException {
-                InputStream is = new URL(u).openConnection().getInputStream();
-                InputSource source = new InputSource(is);
-                                
-                return new SyndFeedInput().build(source);
+                XmlReader r;
+                
+                r = new XmlReader(new URL(u));
+                XmlReader.setDefaultEncoding("UTF-8");
                 
+                return new SyndFeedInput().build(r);                
         }
         
         private static Map<Category, String[]> getFeeds() {
@@ -56,10 +56,11 @@ public class ArticleProvider {
                 result.put(Category.TOP,
                            new String[] {
                                            "http://www.francetvinfo.fr/titres.rss",
-                                           "http://www.france24.com/fr/actualites/rss",
-                                           //"https://www.franceinter.fr/rss/a-la-une.xml",
                                            "http://www.rfi.fr/general/rss",
                                            "http://www.cnews.fr/rss/une",
+                                           "http://www.ladepeche.fr/rss/a-la-une.rss",
+                                           "https://www.franceinter.fr/rss/a-la-une.xml",
+                                           "https://www.francebleu.fr/rss/a-la-une.xml",
                                            "http://www.bfmtv.com/rss/info/flux-rss/flux-toutes-les-actualites/"
                            });
                 
@@ -72,6 +73,10 @@ public class ArticleProvider {
                 
                 result.put(Category.EUROPE,
                                 new String[] { "http://www.france24.com/fr/europe/rss" });
+
+                result.put(Category.MONDE, 
+                           new String[] { "http://www.france24.com/fr/actualites/rss" });                                           
+
                 
                 result.put(Category.ECO,
                                 new String[] { "http://www.france24.com/fr/economie/rss",
@@ -79,8 +84,7 @@ public class ArticleProvider {
                 
                 result.put(Category.ESSONNE,
                                 new String[] { "http://www.tourisme-essonne.com/rss/actus/",
-                                               "http://www.ville-palaiseau.fr/rss/actualites.htm"
-                                                /*"https://www.essonneinfo.fr/feed/"*/ });
+                                               "http://www.ville-palaiseau.fr/rss/actualites.htm" });
                 
                 result.put(Category.PEOPLE,
                                 new String[] { "http://www.premiere.fr/rss/actu-live",
@@ -96,12 +100,19 @@ public class ArticleProvider {
         
         private void addArticles(Category cat, SyndFeed feed) {
                 String thumbnail;
-                String desc, link, title;
+                String desc, link, title, feedTitle, feedImage;
                 Date date;
                 List<Article> articles;
                 boolean exist;
                 
-                LOG.info("addArticles" + cat.getId());
+                feedTitle = feed.getTitle().trim();
+                
+                if (feed.getImage() != null)
+                        feedImage = feed.getImage().getUrl();
+                else
+                        feedImage = null;
+                
+                LOG.info("addArticles " + cat.getId() + " " + feedTitle + " number of articles: " + feed.getEntries().size());
                 
                 for (SyndEntry entry: feed.getEntries()) {
                         thumbnail = null;
@@ -110,21 +121,29 @@ public class ArticleProvider {
                                         thumbnail = e.getUrl();    
                                 break;
                         }
-                                
+                        if (thumbnail == null)
+                                thumbnail = feedImage;
+                                                        
+                        title = entry.getTitle().trim();
+                        
                         if (entry.getDescription() != null) {                                      
                                 desc = Jsoup.parse(entry.getDescription().getValue()).text();
                         } else {       
                                 desc = null;
-                                LOG.severe("No description for " + feed.getTitle() + " - " + entry.getTitle());
+                                LOG.severe("No description for " + feedTitle + " - " + title);
                         }
                         
                         date = entry.getPublishedDate();
                         if (date == null)
                                 date = entry.getUpdatedDate();
+                        if (date == null) {
+                                LOG.severe("The article " + feedTitle + " - " + title + " does not have a date");
+                                continue;
+                        }                                
                         
                         synchronized(articlesByCategory) {
                                 link = entry.getLink().trim();
-                                title = entry.getTitle().trim();
+
                                 articles = articlesByCategory.get(cat);
                                 exist = false;
                                 if (articles == null) {
@@ -133,13 +152,13 @@ public class ArticleProvider {
                                 } else {                                
                                         for (Article a: articles)
                                                 if (a.link.equals(link)) {
-                                                        LOG.finest(link + " already present");
+                                                        LOG.fine("addArticles " + link + " is already present");
                                                         exist = true;
                                                 }
                                 }
                                 
                                 if (!exist) {
-                                        LOG.info("add " + cat.getId() + " " + feed.getTitle() + " " + title);
+                                        LOG.fine("add " + cat.getId() + " " + feedTitle + " " + title);
                                 
                                         articles.add(new Article(link, cat, title, desc, thumbnail, date,
                                                         feed.getTitle()));
@@ -151,9 +170,13 @@ public class ArticleProvider {
                                                 }
 
                                         });
+                                } else {
+                                        LOG.finest("addArticles already exist: " + title);
                                 }
                         }
-                }               
+                }          
+                
+                LOG.info("addArticles done " + cat.getId());
         }
         
         private void retrieveArticles(Category cat) throws IllegalArgumentException, MalformedURLException, FeedException, IOException {
@@ -165,17 +188,22 @@ public class ArticleProvider {
                         for (String str: feeds)
                                 try {
                                         addArticles(cat, getSyndFeed(str));
-                                } catch (IOException e) {
-                                        LOG.log(Level.SEVERE, "retrieveArticles failure " + cat.getId(), e);
+                                } catch (Throwable e) {
+                                        LOG.log(Level.SEVERE,
+                                                "retrieveArticles failure " + cat.getId() + " " + str,
+                                                e);
                                 }
                 else
                         LOG.severe("No feed for category " + cat);
         }
         
+        /**
+         * Returns a copy.
+         */
         public List<Article> getArticles(Category cat)
                         throws IllegalArgumentException, MalformedURLException, FeedException, IOException {
                 synchronized (articlesByCategory) {
-                        return articlesByCategory.get(cat);
+                        return new ArrayList<>(articlesByCategory.get(cat));
                 }
         }
         
@@ -202,6 +230,7 @@ public class ArticleProvider {
                                                                        articles.subList(0, 100));
                                                                 
                                         }
+                                        LOG.info("refresher " + category.getId() + " number of articles: " + articles.size());
                                 }
                         } catch (IllegalArgumentException | FeedException | IOException e) {
                                 LOG.log(Level.SEVERE, "refresher failure", e);