From 6225887c0f8a1c22754ac98941908f47281bb2d3 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Orsini Date: Thu, 12 Oct 2017 16:22:35 +0200 Subject: [PATCH] updated --- pnews/pom.xml | 4 +- pnews/src/main/java/pnews/HTML.java | 4 +- pnews/src/main/java/pnews/Main.java | 203 --------------------- pnews/style.css | 74 -------- war/pom.xml | 21 +++ .../main/java/pnews/servlet/ArticleProvider.java | 193 ++++++++++++++++++++ war/src/main/java/pnews/servlet/Pnews.java | 86 ++++++++- war/src/main/webapp/WEB-INF/web.xml | 5 + 8 files changed, 304 insertions(+), 286 deletions(-) delete mode 100644 pnews/src/main/java/pnews/Main.java delete mode 100644 pnews/style.css create mode 100644 war/src/main/java/pnews/servlet/ArticleProvider.java diff --git a/pnews/pom.xml b/pnews/pom.xml index 5631d03..ee5c156 100644 --- a/pnews/pom.xml +++ b/pnews/pom.xml @@ -2,10 +2,10 @@ 4.0.0 pnews - pnews + pnews-core 1.0 jar - pnews + pnews-core UTF-8 diff --git a/pnews/src/main/java/pnews/HTML.java b/pnews/src/main/java/pnews/HTML.java index 78d584d..74df5e7 100644 --- a/pnews/src/main/java/pnews/HTML.java +++ b/pnews/src/main/java/pnews/HTML.java @@ -38,7 +38,7 @@ public class HTML { buf.append(a.thumbnail); buf.append("'/>\n"); } - appendA(buf, a.title, a.link, null); + appendA(buf, a.title, "redirect?url="+a.link, null); buf.append("\n"); buf.append("
" + a.website + " - " + a.publicationDate + "
"); @@ -66,7 +66,7 @@ public class HTML { else cl = null; - appendA(buf, cat.getId(), cat.getId() + ".html", cl); + appendA(buf, cat.getId(), cat.getId(), cl); buf.append(""); } diff --git a/pnews/src/main/java/pnews/Main.java b/pnews/src/main/java/pnews/Main.java deleted file mode 100644 index 6086a74..0000000 --- a/pnews/src/main/java/pnews/Main.java +++ /dev/null @@ -1,203 +0,0 @@ -package pnews; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.security.KeyManagementException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.X509Certificate; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.logging.Logger; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; - -import org.jsoup.Jsoup; - -import com.rometools.rome.feed.synd.SyndEnclosure; -import com.rometools.rome.feed.synd.SyndEntry; -import com.rometools.rome.feed.synd.SyndFeed; -import com.rometools.rome.io.FeedException; -import com.rometools.rome.io.SyndFeedInput; -import com.rometools.rome.io.XmlReader; - -public class Main { - private static final Logger LOG = Logger.getLogger(Main.class.getName()); - - static { - TrustManager[] mgrs; - SSLContext sc; - - mgrs = new TrustManager[]{ - new X509TrustManager() { - public java.security.cert.X509Certificate[] getAcceptedIssuers() { - return null; - } - - public void checkClientTrusted(X509Certificate[] certs, String authType) { - } - - public void checkServerTrusted(X509Certificate[] certs, String authType) { - } - } - }; - - try { - sc = SSLContext.getInstance("SSL"); - - sc.init(null, mgrs, new java.security.SecureRandom()); - SSLContext.setDefault(sc); - } catch (NoSuchAlgorithmException | KeyManagementException e) { - e.printStackTrace(); - } - } - - private static void addArticles(Category cat, SyndFeed feed, List
articles) { - String thumbnail; - String desc; - Date date; - - for (SyndEntry entry: feed.getEntries()) { - thumbnail = null; - for (SyndEnclosure e: entry.getEnclosures()) { - if (e.getType().startsWith("image/")) - thumbnail = e.getUrl(); - break; - } - - if (entry.getDescription() != null) { - desc = Jsoup.parse(entry.getDescription().getValue()).text(); - } else { - desc = null; - LOG.severe("No description for " + feed.getTitle() + " - " + entry.getTitle()); - } - - date = entry.getPublishedDate(); - if (date == null) - date = entry.getUpdatedDate(); - - articles.add(new Article(entry.getLink(), - cat, - entry.getTitle(), - desc, - thumbnail, - date, - feed.getTitle())); - } - } - - private static SyndFeed getSyndFeed(String u) throws IllegalArgumentException, FeedException, MalformedURLException, IOException { - try (XmlReader reader = new XmlReader(new URL(u))) { - return new SyndFeedInput().build(reader); - } - } - - private static Map getFeeds() { - Map result; - - result = new HashMap<>(); - - result.put(Category.TOP, - new String[] { - "http://www.francetvinfo.fr/titres.rss", - "http://www.france24.com/fr/actualites/rss", - "https://www.franceinter.fr/rss/a-la-une.xml", - "http://www.rfi.fr/general/rss", - "http://www.cnews.fr/rss/une", - "http://www.bfmtv.com/rss/info/flux-rss/flux-toutes-les-actualites/" - }); - - result.put(Category.SPORT, - new String[] { "http://www.france24.com/fr/sports/rss" }); - - result.put(Category.FRANCE, - new String[] { "http://www.france24.com/fr/france/rss", - "http://www.rfi.fr/france/rss"}); - - result.put(Category.EUROPE, - new String[] { "http://www.france24.com/fr/europe/rss" }); - - result.put(Category.ECO, - new String[] { "http://www.france24.com/fr/economie/rss", - "http://www.rfi.fr/economie/rss" }); - - result.put(Category.ESSONNE, - new String[] { "https://www.essonneinfo.fr/feed/" }); - - result.put(Category.TECHNOLOGIE, - new String[] { "http://feeds.feedburner.com/lesnumeriques/news", - "http://www.zdnet.fr/feeds/rss/actualites/"}); - - return result; - } - - private static List
getArticles(Category cat) throws IllegalArgumentException, MalformedURLException, FeedException, IOException { - List
articles; - String[] feeds; - Set links; - - articles = new ArrayList<>(); - - feeds = getFeeds().get(cat); - - if (feeds != null) - for (String str: feeds) - addArticles(cat, getSyndFeed(str), articles); - else - LOG.severe("No feed for category " + cat); - - links = new HashSet<>(articles.size()); - for (Article a: articles) { - if (links.contains(a.link)) - LOG.severe(a.link + "is not uniq"); - else - links.add(a.link); - } - - articles.sort(new Comparator
() { - @Override - public int compare(Article o1, Article o2) { - return o2.publicationDate.compareTo(o1.publicationDate); - } - }); - - return articles; - } - - private static void writeHTMLFile(Category cat) throws IllegalArgumentException, MalformedURLException, FeedException, IOException { - List
articles; - String html; - File f; - - articles = getArticles(cat); - - html = HTML.toHTML(articles, cat); - - f = new File(cat.getId() + ".html"); - - try (BufferedWriter writer = Files.newBufferedWriter(f.toPath(), StandardCharsets.UTF_8)) { - writer.write(html); - } - } - - public static void main(String[] args) throws IllegalArgumentException, FeedException, IOException { - System.out.println("pnews"); - - for (Category cat: Category.values()) - writeHTMLFile(cat); - - System.out.println("done"); - } -} \ No newline at end of file diff --git a/pnews/style.css b/pnews/style.css deleted file mode 100644 index cc79235..0000000 --- a/pnews/style.css +++ /dev/null @@ -1,74 +0,0 @@ -a { - text-decoration: none; - color: black; -} - -body { - margin: 0 0 0 0; - padding: 1em 1em 1em 1em; - background-color: #eee; - font-family: sans-serif; - font-size: 100%; -} - -nav { - font-size: 125%; - margin: 0 0 0 0; - padding: 0 0 0 0; -} - -a.active { - text-decoration: none; - border-bottom: 4px solid black; -} - -div { - margin: 0em 0em 0em 0em; - padding: 0 0 0 0; -} - -div.article { - margin-bottom: 1em; -} - -.article-info { - font-size: 80%; - color: #bbb; -} - -img { - margin: 0em 1em 1em 0em; - padding: 0 0 0 0; - width: 8em; -} - -p { - margin: 1em 1em 1em 1em; - padding: 0 0 0 0; -} - -.left { - float: left; -} - -h2 { - clear: left; - margin: 0 0 0 0; - padding: 0 0 0 0; -} - -nav { - font-size: 120%; -} - -nav ul { - list-style-type: none; - padding: 0 0 0 0; -} - -nav ul li { - display: inline; - margin: 0em 1em 0 0; - padding: 0 0 0 0; - text-transform: uppercase; -} diff --git a/war/pom.xml b/war/pom.xml index aaa43c7..7f1e992 100644 --- a/war/pom.xml +++ b/war/pom.xml @@ -6,6 +6,12 @@ war pnews + + UTF-8 + 1.8 + 1.8 + + javax.servlet @@ -13,5 +19,20 @@ 3.1.0 provided + + pnews + pnews-core + 1.0 + + + com.rometools + rome + 1.8.0 + + + org.jsoup + jsoup + 1.10.3 + diff --git a/war/src/main/java/pnews/servlet/ArticleProvider.java b/war/src/main/java/pnews/servlet/ArticleProvider.java new file mode 100644 index 0000000..95e3951 --- /dev/null +++ b/war/src/main/java/pnews/servlet/ArticleProvider.java @@ -0,0 +1,193 @@ +package pnews.servlet; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.jsoup.Jsoup; + +import com.rometools.rome.feed.synd.SyndEnclosure; +import com.rometools.rome.feed.synd.SyndEntry; +import com.rometools.rome.feed.synd.SyndFeed; +import com.rometools.rome.io.FeedException; +import com.rometools.rome.io.SyndFeedInput; +import com.rometools.rome.io.XmlReader; + +import pnews.Article; +import pnews.Category; + +public class ArticleProvider { + public final static ArticleProvider singleton = new ArticleProvider(); + private static final Logger LOG = Logger.getLogger(ArticleProvider.class.getName()); + private final Map> articlesByCategory = new HashMap<>(); + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(2); + + private ArticleProvider() { + for (Category cat:Category.values()) + scheduler.scheduleAtFixedRate(new Refresher(cat), 2, 120, TimeUnit.SECONDS); + } + + private static SyndFeed getSyndFeed(String u) throws IllegalArgumentException, FeedException, MalformedURLException, IOException { + try (XmlReader reader = new XmlReader(new URL(u))) { + return new SyndFeedInput().build(reader); + } + } + + private static Map getFeeds() { + Map result; + + result = new HashMap<>(); + + result.put(Category.TOP, + new String[] { + "http://www.francetvinfo.fr/titres.rss", + "http://www.france24.com/fr/actualites/rss", + //"https://www.franceinter.fr/rss/a-la-une.xml", + "http://www.rfi.fr/general/rss", + "http://www.cnews.fr/rss/une", + "http://www.bfmtv.com/rss/info/flux-rss/flux-toutes-les-actualites/" + }); + + result.put(Category.SPORT, + new String[] { "http://www.france24.com/fr/sports/rss" }); + + result.put(Category.FRANCE, + new String[] { "http://www.france24.com/fr/france/rss", + "http://www.rfi.fr/france/rss"}); + + result.put(Category.EUROPE, + new String[] { "http://www.france24.com/fr/europe/rss" }); + + result.put(Category.ECO, + new String[] { "http://www.france24.com/fr/economie/rss", + "http://www.rfi.fr/economie/rss" }); + + result.put(Category.ESSONNE, + new String[] { /*"https://www.essonneinfo.fr/feed/"*/ }); + + result.put(Category.TECHNOLOGIE, + new String[] { "http://feeds.feedburner.com/lesnumeriques/news", + "http://www.zdnet.fr/feeds/rss/actualites/"}); + + return result; + } + + private void addArticles(Category cat, SyndFeed feed) { + String thumbnail; + String desc; + Date date; + List
articles; + + LOG.info("addArticles" + cat.getId()); + + for (SyndEntry entry: feed.getEntries()) { + thumbnail = null; + for (SyndEnclosure e: entry.getEnclosures()) { + if (e.getType().startsWith("image/")) + thumbnail = e.getUrl(); + break; + } + + if (entry.getDescription() != null) { + desc = Jsoup.parse(entry.getDescription().getValue()).text(); + } else { + desc = null; + LOG.severe("No description for " + feed.getTitle() + " - " + entry.getTitle()); + } + + date = entry.getPublishedDate(); + if (date == null) + date = entry.getUpdatedDate(); + + synchronized(articlesByCategory) { + articles = articlesByCategory.get(cat); + if (articles == null) { + articles = new ArrayList<>(); + articlesByCategory.put(cat, articles); + } else { + for (Article a: articles) + if (a.link.equals(entry.getLink())) + return ; + } + + articles.add(new Article(entry.getLink(), + cat, + entry.getTitle(), + desc, + thumbnail, + date, + feed.getTitle())); + + articles.sort(new Comparator
() { + @Override + public int compare(Article o1, Article o2) { + return o2.publicationDate.compareTo(o1.publicationDate); + } + + }); + } + } + } + + private void retrieveArticles(Category cat) throws IllegalArgumentException, MalformedURLException, FeedException, IOException { + String[] feeds; + + feeds = getFeeds().get(cat); + + if (feeds != null) + for (String str: feeds) + addArticles(cat, getSyndFeed(str)); + else + LOG.severe("No feed for category " + cat); + } + + public List
getArticles(Category cat) + throws IllegalArgumentException, MalformedURLException, FeedException, IOException { + synchronized (articlesByCategory) { + return articlesByCategory.get(cat); + } + } + + private class Refresher implements Runnable { + private final Category category; + + public Refresher(Category category) { + this.category = category; + } + + @Override + public void run() { + List
articles; + + LOG.info("refresher "+ category.getId()); + + try { + retrieveArticles(category); + + synchronized (articlesByCategory) { + articles = articlesByCategory.get(category); + if (articles != null && articles.size() > 100) { + articlesByCategory.put(category, + articles.subList(0, 100)); + + } + } + } catch (IllegalArgumentException | FeedException | IOException e) { + LOG.log(Level.SEVERE, "refresher failure", e); + } + + LOG.info("refresher "+ category.getId() + " done"); + } + } +} diff --git a/war/src/main/java/pnews/servlet/Pnews.java b/war/src/main/java/pnews/servlet/Pnews.java index f7175a1..d15315e 100644 --- a/war/src/main/java/pnews/servlet/Pnews.java +++ b/war/src/main/java/pnews/servlet/Pnews.java @@ -1,8 +1,14 @@ package pnews.servlet; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; import java.io.UnsupportedEncodingException; +import java.io.Writer; import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -12,10 +18,17 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import com.rometools.rome.io.FeedException; + +import pnews.Article; +import pnews.Category; +import pnews.HTML; + public class Pnews extends HttpServlet { private static final Logger LOG = Logger.getLogger(Pnews.class.getName()); private static final long serialVersionUID = 1L; - + private static final ArticleProvider provider = ArticleProvider.singleton; + private static String getQueryParameter(HttpServletRequest rq, String key) throws UnsupportedEncodingException { String[] params; @@ -42,6 +55,8 @@ public class Pnews extends HttpServlet { private static void redirect(HttpServletRequest rq, HttpServletResponse rp) { String redirectURL; + LOG.entering(Pnews.class.getName(), "redirect"); + try { redirectURL = getQueryParameter(rq, "url"); @@ -51,6 +66,7 @@ public class Pnews extends HttpServlet { rp.setHeader("Location", redirectURL); rp.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT); } else { + LOG.severe("No redirection URL"); rp.setStatus(HttpServletResponse.SC_NOT_FOUND); } @@ -59,11 +75,47 @@ public class Pnews extends HttpServlet { LOG.log(Level.SEVERE, "redirect failure", e); rp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); } + + LOG.exiting(Pnews.class.getName(), "redirect"); + } + + private void writeArticles(Category cat, HttpServletResponse rp) { + String html; + List
articles; + + try { + articles = provider.getArticles(cat); + if (articles != null) { + html = HTML.toHTML(articles, cat); + rp.setContentType("text/html"); + rp.getWriter().write(html); + } else { + LOG.severe("writeArticles cannot retrieve any articles"); + html = HTML.toHTML(new ArrayList<>(), cat); + rp.setContentType("text/html"); + rp.getWriter().write(html); + } + } catch (IOException | IllegalArgumentException | FeedException e) { + LOG.log(Level.SEVERE, "writeArticles failure", e); + rp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + } + + private void copy(InputStream in, Writer writer) throws IOException { + Reader r; + char[] buf; + int n; + + buf = new char[1024]; + r = new InputStreamReader(in); + while ( (n = r.read(buf, 0, buf.length)) != -1) + writer.write(buf, 0, n); } @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) { + protected void doGet(HttpServletRequest req, HttpServletResponse resp) { String path; + InputStream in; LOG.info("doGet " + req.getRequestURI()); @@ -71,14 +123,38 @@ public class Pnews extends HttpServlet { if (path.equals("/redirect")) { redirect(req, resp); - } else { + return ; + } + + if (path.equals("/style.css")) { try { - resp.getWriter().write("Not found " + req.getPathInfo()); - resp.setStatus(HttpServletResponse.SC_NOT_FOUND); + in = HTML.class.getClassLoader().getResourceAsStream("style.css"); + copy(in, resp.getWriter()); + resp.setContentType("text/css"); + + return ; } catch (IOException e) { LOG.log(Level.SEVERE, "doGet failure", e); + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + + return ; } } + + for (Category cat: Category.values()) { + if (path.equals('/' + cat.getId())) { + writeArticles(cat, resp); + return ; + } + } + + try { + resp.getWriter().write("Not found " + req.getPathInfo()); + resp.setStatus(HttpServletResponse.SC_NOT_FOUND); + } catch (IOException e) { + LOG.log(Level.SEVERE, "doGet failure", e); + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } } @Override diff --git a/war/src/main/webapp/WEB-INF/web.xml b/war/src/main/webapp/WEB-INF/web.xml index 80d7c74..3c47adf 100644 --- a/war/src/main/webapp/WEB-INF/web.xml +++ b/war/src/main/webapp/WEB-INF/web.xml @@ -11,6 +11,11 @@ 1 + + css + text/css + + pnews /* -- 2.7.4