3 import java.io.IOException;
4 import java.net.MalformedURLException;
6 import java.util.ArrayList;
7 import java.util.Collections;
8 import java.util.Comparator;
10 import java.util.HashMap;
11 import java.util.List;
13 import java.util.concurrent.Executors;
14 import java.util.concurrent.ScheduledExecutorService;
15 import java.util.concurrent.TimeUnit;
16 import java.util.logging.Level;
17 import java.util.logging.Logger;
19 import org.jsoup.Jsoup;
21 import com.rometools.rome.feed.synd.SyndEnclosure;
22 import com.rometools.rome.feed.synd.SyndEntry;
23 import com.rometools.rome.feed.synd.SyndFeed;
24 import com.rometools.rome.io.FeedException;
25 import com.rometools.rome.io.SyndFeedInput;
26 import com.rometools.rome.io.XmlReader;
29 import pnews.Category;
31 public class ArticleProvider {
32 public final static ArticleProvider singleton = new ArticleProvider();
33 private static final Logger LOG = Logger.getLogger(ArticleProvider.class.getName());
34 private final Map<Category, List<Article>> articlesByCategory = new HashMap<>();
35 private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(2);
37 private ArticleProvider() {
38 for (Category cat:Category.values())
39 scheduler.scheduleAtFixedRate(new Refresher(cat), 2, 600, TimeUnit.SECONDS);
42 private static SyndFeed getSyndFeed(String u) throws IllegalArgumentException, FeedException, MalformedURLException, IOException {
45 r = new XmlReader(new URL(u));
47 return new SyndFeedInput().build(r);
52 private List<Article> getArticlesForUpdate(Category cat) {
55 synchronized (articlesByCategory) {
56 result = articlesByCategory.get(cat);
58 result = new ArrayList<>();
59 articlesByCategory.put(cat, result);
65 private boolean exists(String articleLink, List<Article> articles) {
66 synchronized (articles) {
67 for (Article a: articles)
68 if (a.link.equals(articleLink))
74 private static Article toArticle(String link, SyndEntry entry, SyndFeed feed) {
75 String desc, title, thumbnail, feedTitle, str;
78 feedTitle = feed.getTitle();
79 if (feedTitle != null) {
80 feedTitle = feedTitle.trim();
84 for (SyndEnclosure e: entry.getEnclosures()) {
85 if (e.getType().startsWith("image/"))
86 thumbnail = e.getUrl();
90 if (thumbnail == null && feed.getImage() != null)
91 thumbnail = feed.getImage().getUrl();
94 title = entry.getTitle().trim();
96 if (entry.getDescription() != null) {
97 str = entry.getDescription().getValue();
98 desc = Jsoup.parse(str).text();
101 LOG.severe("No description for " + feedTitle + " - " + title);
104 date = entry.getPublishedDate();
106 date = entry.getUpdatedDate();
108 LOG.severe("The article " + feedTitle + " - " + title + " does not have a date");
110 return new Article(link, title, desc, thumbnail, date, feedTitle);
113 private void addArticles(Category cat, SyndFeed feed) {
115 List<Article> articles;
118 feedTitle = feed.getTitle().trim();
120 LOG.info("addArticles " + cat.getId() + " " + feedTitle + " number of articles: " + feed.getEntries().size());
122 for (SyndEntry entry: feed.getEntries()) {
123 String link = entry.getLink().trim();
124 articles = getArticlesForUpdate(cat);
125 if (exists(link, articles)) {
126 LOG.fine("addArticles " + link + " is already present");
130 a = ArticleStore.singleton.getArticle(link, ()->toArticle(link, entry, feed));
132 synchronized (articles) {
135 Collections.sort(articles, new Comparator<Article>() {
137 public int compare(Article o1, Article o2) {
138 if (o1.publicationDate == o2.publicationDate)
140 if (o1.publicationDate == null)
142 if (o2.publicationDate == null)
144 return o2.publicationDate.compareTo(o1.publicationDate);
150 LOG.info("addArticles done " + cat.getId());
153 private void retrieveArticles(Category cat) throws IllegalArgumentException, MalformedURLException, FeedException, IOException {
156 feeds = Config.getFeedsByCategory().get(cat);
159 for (String str: feeds)
161 addArticles(cat, getSyndFeed(str));
162 } catch (Throwable e) {
163 LOG.log(Level.SEVERE,
164 "retrieveArticles failure " + cat.getId() + " " + str,
168 LOG.severe("No feed for category " + cat);
174 public List<Article> getArticles(Category cat)
175 throws IllegalArgumentException, MalformedURLException, FeedException, IOException {
176 List<Article> articles;
178 synchronized (articlesByCategory) {
179 articles = getArticlesForUpdate(cat);
182 synchronized (articles) {
183 return new ArrayList<>(articles);
187 private class Refresher implements Runnable {
188 private final Category category;
190 public Refresher(Category category) {
191 this.category = category;
196 LOG.info("refresher "+ category.getId());
199 retrieveArticles(category);
200 } catch (IllegalArgumentException | FeedException | IOException e) {
201 LOG.log(Level.SEVERE, "refresher failure", e);
204 LOG.info("refresher "+ category.getId() + " done");