From e28fb81c2f705d5bc47464ccbf08481a06b1035d Mon Sep 17 00:00:00 2001 From: Jean-Philippe Orsini Date: Sun, 5 Nov 2017 22:35:51 +0100 Subject: [PATCH] do not load articles older than 60 days --- war/src/main/java/pnews/Article.java | 8 ++--- .../main/java/pnews/servlet/ArticleProvider.java | 34 +++++++++++++++------- war/src/main/java/pnews/servlet/Config.java | 15 ++++++++++ war/src/main/resources/feeds.json | 5 +++- 4 files changed, 46 insertions(+), 16 deletions(-) diff --git a/war/src/main/java/pnews/Article.java b/war/src/main/java/pnews/Article.java index 97e8b76..5b2e21c 100644 --- a/war/src/main/java/pnews/Article.java +++ b/war/src/main/java/pnews/Article.java @@ -1,6 +1,6 @@ package pnews; -import java.util.Date; +import java.time.Instant; import java.util.concurrent.atomic.AtomicLong; public class Article { @@ -8,12 +8,12 @@ public class Article { public final String description; public final String thumbnail; public final String link; - public final Date publicationDate; + public final Instant publicationDate; public final String website; public final String[] entities; public final AtomicLong readCount = new AtomicLong(); - public Article(String link, String title, String description, String thumbnail, Date publicationDate, String website, String[] entities) { + public Article(String link, String title, String description, String thumbnail, Instant publicationDate, String website, String[] entities) { this.link = link; this.title = title; this.description = description; @@ -35,7 +35,7 @@ public class Article { return false; } - public Date getPublicationDate() { + public Instant getPublicationDate() { return publicationDate; } } diff --git a/war/src/main/java/pnews/servlet/ArticleProvider.java b/war/src/main/java/pnews/servlet/ArticleProvider.java index ec74123..5d4959b 100644 --- a/war/src/main/java/pnews/servlet/ArticleProvider.java +++ b/war/src/main/java/pnews/servlet/ArticleProvider.java @@ -3,6 +3,7 @@ package pnews.servlet; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -74,7 +75,20 @@ public class ArticleProvider { return false; } - private Article toArticle(String link, SyndEntry entry, SyndFeed feed, String lang) { + private Instant getArticleInstant(SyndEntry entry) { + Date date; + + date = entry.getUpdatedDate(); + if (date == null) + date = entry.getPublishedDate(); + + if (date == null) + return Instant.now(); + + return date.toInstant(); + } + + private Article toArticle(String link, SyndEntry entry, SyndFeed feed, String lang, Instant instant) { String desc, title, thumbnail, feedTitle, str; Date date; List entities; @@ -100,14 +114,7 @@ public class ArticleProvider { desc = null; LOG.severe("No description for " + feedTitle + " - " + title); } - - date = entry.getPublishedDate(); - if (date == null) - date = entry.getUpdatedDate(); - if (date == null) - LOG.severe("The article " + feedTitle + " - " + title + " does not have a date"); - - + entities = new ArrayList<>(); if (desc != null && lang.equals("en")) try { @@ -117,7 +124,7 @@ public class ArticleProvider { LOG.log(Level.SEVERE, "Cannot classify " + feedTitle, e1); } - return new Article(link, title, desc, thumbnail, date, feedTitle, entities.toArray(new String[0])); + return new Article(link, title, desc, thumbnail, instant, feedTitle, entities.toArray(new String[0])); } private void addArticles(Category cat, SyndFeed feed) { @@ -137,7 +144,12 @@ public class ArticleProvider { continue ; } - a = ArticleStore.singleton.getArticle(link, ()->toArticle(link, entry, feed, cat.getLanguage())); + final Instant instant = getArticleInstant(entry); + + if (config.isObsolete(instant)) + continue ; + + a = ArticleStore.singleton.getArticle(link, ()->toArticle(link, entry, feed, cat.getLanguage(), instant)); synchronized (articles) { articles.add(a); diff --git a/war/src/main/java/pnews/servlet/Config.java b/war/src/main/java/pnews/servlet/Config.java index 46e75f6..36bdc62 100644 --- a/war/src/main/java/pnews/servlet/Config.java +++ b/war/src/main/java/pnews/servlet/Config.java @@ -4,6 +4,9 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; import java.io.UnsupportedEncodingException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalUnit; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -19,6 +22,7 @@ import javax.json.JsonObject; import javax.json.JsonString; import javax.json.JsonValue; +import pnews.Article; import pnews.Category; import pnews.Feed; import pnews.Language; @@ -180,6 +184,17 @@ public class Config { return result; } + public boolean isObsolete(Instant instant) { + Instant olderInstant; + + olderInstant = Instant.now().minus(60, ChronoUnit.DAYS); + + if (instant.isAfter(olderInstant)) + return false; + else + return true; + } + public Feed[] getFeeds() { return feeds; } diff --git a/war/src/main/resources/feeds.json b/war/src/main/resources/feeds.json index b10615c..03d4217 100644 --- a/war/src/main/resources/feeds.json +++ b/war/src/main/resources/feeds.json @@ -130,6 +130,7 @@ "http://www.markshuttleworth.com/feed": { "categories": ["en_linux"]}, "https://insights.ubuntu.com/feed/": { "categories": ["en_linux"]}, "http://feeds.feedburner.com/LinuxJournal-BreakingNews?format=xml": { "categories": ["en_linux"]}, + "https://www.ossblog.org/feed/": { "categories": ["en_linux"]}, "https://www.gnome.org/feed/": { "categories": ["en_linux"]}, "http://linuxreviews.org/en.rss": { "categories": ["en_linux"]}, "http://www.linux-magazine.com/rss/feed/lmi_news": { "categories": ["en_linux"]}, @@ -151,7 +152,9 @@ "Digital Trends", "Joey Sneddon", "CA", - "Read" + "Read", + "5 :", + "InfoWorld" ], "aliases": { "U.S.": ["United States", "US"], -- 2.7.4