Merge branch 'dev'

# Conflicts: # src/main/java/org/decaywood/utils/HttpRequestHelper.java
wangding75 · Apr 10, 2016 · 9d3c31c · 9d3c31c
2 parents b3acab1 + 74a16fc
commit 9d3c31c
Show file tree

Hide file tree

Showing 10 changed files with 115 additions and 39 deletions.
diff --git a/src/main/java/org/decaywood/CookieProcessor.java b/src/main/java/org/decaywood/CookieProcessor.java
@@ -0,0 +1,67 @@
+package org.decaywood;
+
+import org.decaywood.utils.FileLoader;
+import org.decaywood.utils.RequestParaBuilder;
+
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+/**
+ * @author: decaywood
+ * @date: 2016/04/10 20:01
+ */
+public interface CookieProcessor {
+
+
+    default void updateCookie(String website) throws Exception {
+
+        GlobalSystemConfigLoader.loadConfig();
+
+        String areacode = System.getProperty("areaCode");
+        String userID = System.getProperty("userID");
+        String passwd = System.getProperty("password");
+        boolean rememberMe = Boolean.parseBoolean(System.getProperty("rememberMe"));
+
+        HttpURLConnection connection = null;
+        if (userID != null && passwd != null) {
+            connection = login(areacode, userID, passwd, rememberMe);
+        }
+        try {
+            connection = connection == null ?
+                    (HttpURLConnection) new URL(website).openConnection() : connection;
+            connection.connect();
+
+            String cookie = connection.getHeaderFields().get("Set-Cookie")
+                    .stream()
+                    .map(x -> x.split(";")[0].concat(";"))
+                    .filter(x -> x.contains("token=") || x.contains("s="))
+                    .reduce("", String::concat);
+            FileLoader.updateCookie(cookie, website);
+        } finally {
+            if (connection != null) connection.disconnect();
+        }
+
+    }
+
+    default HttpURLConnection login(String areacode,
+                                    String userID,
+                                    String passwd,
+                                    boolean rememberMe) throws Exception {
+
+        areacode = areacode == null ? "86" : areacode;
+        if (userID == null || passwd == null) {
+            throw new IllegalArgumentException("null parameter: userID or password");
+        }
+
+        RequestParaBuilder builder = new RequestParaBuilder("http://xueqiu.com/user/login")
+                .addParameter("areacode", areacode)
+                .addParameter("telephone", userID)
+                .addParameter("password", passwd)
+                .addParameter("remember_me", rememberMe ? "on" : "off");
+
+        URL url = new URL(builder.build());
+        return (HttpURLConnection) url.openConnection();
+    }
+
+
+}
diff --git a/src/main/java/org/decaywood/acceptor/AbstractAcceptor.java b/src/main/java/org/decaywood/acceptor/AbstractAcceptor.java
@@ -1,9 +1,9 @@
 package org.decaywood.acceptor;
 
 import org.decaywood.AbstractRemoteService;
+import org.decaywood.CookieProcessor;
 import org.decaywood.remote.RemoteAcceptor;
 import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy;
-import org.decaywood.utils.HttpRequestHelper;
 import org.decaywood.utils.URLMapper;
 
 import java.io.IOException;
@@ -20,7 +20,11 @@
 /**
  * 接受处理完的数据流并进行分析，既可以当作整个生命周期的中点也可以当作中间组件使用
  */
-public abstract class AbstractAcceptor<T> extends AbstractRemoteService implements Consumer<T>, Function<T, T>, RemoteAcceptor<T> {
+public abstract class AbstractAcceptor<T> extends AbstractRemoteService implements
+        Consumer<T>,
+        Function<T, T>,
+        RemoteAcceptor<T>,
+        CookieProcessor {
 
     public AbstractAcceptor() throws RemoteException {
         this(null);
@@ -36,6 +40,7 @@ public AbstractAcceptor(TimeWaitingStrategy strategy, String webSite) throws Rem
 
     protected abstract void consumLogic(T t) throws Exception;
 
+
     @Override
     public void accept(T t) {
 
@@ -56,7 +61,7 @@ public void accept(T t) {
                     } catch (Exception e) {
                         if(!(e instanceof IOException)) throw e;
                         System.out.println("Acceptor: Network busy Retrying -> " + loopTime + " times");
-                        HttpRequestHelper.updateCookie(webSite);
+                        updateCookie(webSite);
                         this.strategy.waiting(loopTime++);
                     }
                 }

diff --git a/src/main/java/org/decaywood/collector/AbstractCollector.java b/src/main/java/org/decaywood/collector/AbstractCollector.java
@@ -1,9 +1,9 @@
 package org.decaywood.collector;
 
 import org.decaywood.AbstractRemoteService;
+import org.decaywood.CookieProcessor;
 import org.decaywood.remote.RemoteCollector;
 import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy;
-import org.decaywood.utils.HttpRequestHelper;
 import org.decaywood.utils.URLMapper;
 
 import java.io.IOException;
@@ -22,7 +22,10 @@
  * 如果要贡献模块，强烈建议继承此类，它有进行数据收集所需的API
  * 供调用。有完备的超时重传机制以及等待策略
  */
-public abstract class AbstractCollector<T> extends AbstractRemoteService implements Supplier<T>, RemoteCollector<T> {
+public abstract class AbstractCollector<T> extends AbstractRemoteService implements
+        Supplier<T>,
+        RemoteCollector<T>,
+        CookieProcessor {
 
     /**
      * 收集器收集逻辑,由子类实现
@@ -66,7 +69,7 @@ public T get() {
                     } catch (Exception e) {
                         if(!(e instanceof IOException)) throw e;
                         System.out.println("Collector: Network busy Retrying -> " + loopTime + " times");
-                        HttpRequestHelper.updateCookie(webSite);
+                        updateCookie(webSite);
                         this.strategy.waiting(loopTime++);
                     }
                 }

diff --git a/src/main/java/org/decaywood/collector/HuShenNewsRefCollector.java b/src/main/java/org/decaywood/collector/HuShenNewsRefCollector.java
@@ -2,7 +2,6 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy;
-import org.decaywood.utils.HttpRequestHelper;
 import org.decaywood.utils.RequestParaBuilder;
 import org.decaywood.utils.URLMapper;
 
@@ -100,7 +99,7 @@ public List<URL> collectLogic() throws Exception {
                     } catch (Exception e) {
                         if(!(e instanceof IOException)) throw e;
                         System.out.println("Collector: Network busy Retrying -> " + loopTime + " times");
-                        HttpRequestHelper.updateCookie(webSite);
+                        updateCookie(webSite);
                         this.strategy.waiting(loopTime++);
                     }
                 }

diff --git a/src/main/java/org/decaywood/filter/AbstractFilter.java b/src/main/java/org/decaywood/filter/AbstractFilter.java
@@ -1,9 +1,9 @@
 package org.decaywood.filter;
 
 import org.decaywood.AbstractRemoteService;
+import org.decaywood.CookieProcessor;
 import org.decaywood.remote.RemoteFilter;
 import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy;
-import org.decaywood.utils.HttpRequestHelper;
 import org.decaywood.utils.URLMapper;
 
 import java.io.IOException;
@@ -19,7 +19,10 @@
 /**
  * 过滤器，特别是要访问网页的过滤器，可以继承此抽象类
  */
-public abstract class AbstractFilter<T> extends AbstractRemoteService implements Predicate<T>, RemoteFilter<T> {
+public abstract class AbstractFilter<T> extends AbstractRemoteService implements
+        Predicate<T>,
+        RemoteFilter<T>,
+        CookieProcessor {
 
 
     protected abstract boolean filterLogic(T t) throws Exception;
@@ -57,7 +60,7 @@ public boolean test(T t) {
                     } catch (Exception e) {
                         if(!(e instanceof IOException)) throw e;
                         System.out.println("Filter: Network busy Retrying -> " + loopTime + " times");
-                        HttpRequestHelper.updateCookie(webSite);
+                        updateCookie(webSite);
                         this.strategy.waiting(loopTime++);
                     }
                 }

diff --git a/src/main/java/org/decaywood/mapper/AbstractMapper.java b/src/main/java/org/decaywood/mapper/AbstractMapper.java
@@ -1,6 +1,7 @@
 package org.decaywood.mapper;
 
 import org.decaywood.AbstractRemoteService;
+import org.decaywood.CookieProcessor;
 import org.decaywood.remote.RemoteMapper;
 import org.decaywood.entity.DeepCopy;
 import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy;
@@ -17,7 +18,10 @@
  * @date: 2015/11/24 16:56
  */
 
-public abstract class AbstractMapper <T, R> extends AbstractRemoteService implements Function<T, R>, RemoteMapper<T, R> {
+public abstract class AbstractMapper <T, R> extends AbstractRemoteService implements
+        Function<T, R>,
+        RemoteMapper<T, R>,
+        CookieProcessor {
 
 
     protected abstract R mapLogic(T t) throws Exception;
@@ -57,7 +61,7 @@ public R apply(T t) {
                     } catch (Exception e) {
                         if (!(e instanceof IOException)) throw e;
                         System.out.println("Mapper: Network busy Retrying -> " + loopTime + " times");
-                        HttpRequestHelper.updateCookie(webSite);
+                        updateCookie(webSite);
                         this.strategy.waiting(loopTime++);
                     }
                 }

diff --git a/src/main/java/org/decaywood/timeWaitingStrategy/DefaultTimeWaitingStrategy.java b/src/main/java/org/decaywood/timeWaitingStrategy/DefaultTimeWaitingStrategy.java
@@ -10,7 +10,7 @@
  */
 public class DefaultTimeWaitingStrategy implements TimeWaitingStrategy {
 
-    private final long timeWaitingShreshold;
+    private final long timeWaitingThreshold;
     private final long timeWaiting;
 
     private final int retryTime;
@@ -22,12 +22,12 @@ public DefaultTimeWaitingStrategy() {
 
     /**
      *
-     * @param timeWaitingShreshold 超时等待阈值（最多等待阈值指定时间然后进入下一次请求尝试）
+     * @param timeWaitingThreshold 超时等待阈值（最多等待阈值指定时间然后进入下一次请求尝试）
      * @param timeWaiting 起始等待时间
      * @param retryTime 重试次数（超过次数抛出超时异常）
      */
-    public DefaultTimeWaitingStrategy(final long timeWaitingShreshold, long timeWaiting, int retryTime) {
-        this.timeWaitingShreshold = timeWaitingShreshold;
+    public DefaultTimeWaitingStrategy(final long timeWaitingThreshold, long timeWaiting, int retryTime) {
+        this.timeWaitingThreshold = timeWaitingThreshold;
         this.timeWaiting = timeWaiting;
         this.retryTime = retryTime;
     }
@@ -38,7 +38,7 @@ public void waiting(int loopTime) {
         try {
 
             long sleepTime = this.timeWaiting * (2 << loopTime);
-            sleepTime = Math.min(sleepTime, timeWaitingShreshold);
+            sleepTime = Math.min(sleepTime, timeWaitingThreshold);
             Thread.sleep(sleepTime);
 
         } catch (InterruptedException e) {

diff --git a/src/main/java/org/decaywood/utils/HttpRequestHelper.java b/src/main/java/org/decaywood/utils/HttpRequestHelper.java
@@ -56,35 +56,24 @@ public String request(URL url) throws IOException {
 
     public String request(URL url, Map<String, String> config) throws IOException {
         HttpURLConnection httpURLConn = null;
-        try
-        {
-            httpURLConn= (HttpURLConnection)url.openConnection();
-            if(post) httpURLConn.setRequestMethod("Post");
+        try {
+            httpURLConn = (HttpURLConnection) url.openConnection();
+            if (post) httpURLConn.setRequestMethod("POST");
             httpURLConn.setDoOutput(true);
-            for (Map.Entry<String, String> entry : config.entrySet()) httpURLConn.setRequestProperty(entry.getKey(), entry.getValue());
+            for (Map.Entry<String, String> entry : config.entrySet())
+                httpURLConn.setRequestProperty(entry.getKey(), entry.getValue());
             httpURLConn.connect();
-            InputStream in =httpURLConn.getInputStream();
-            if(gzip) in = new GZIPInputStream(in);
+            InputStream in = httpURLConn.getInputStream();
+            if (gzip) in = new GZIPInputStream(in);
             BufferedReader bd = new BufferedReader(new InputStreamReader(in));
             StringBuilder builder = new StringBuilder();
             String text;
-            while((text=bd.readLine())!=null) builder.append(text);
+            while ((text = bd.readLine()) != null) builder.append(text);
             return builder.toString();
+        } finally {
+            if (httpURLConn != null) httpURLConn.disconnect();
         }
-        finally { if(httpURLConn!=null) httpURLConn.disconnect(); }
     }
 
 
-    public static void updateCookie(String website) throws Exception {
-        URL url = new URL(website);
-        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
-        connection.connect();
-        String cookie = connection.getHeaderFields().get("Set-Cookie")
-                .stream()
-                .map(x -> x.split(";")[0].concat(";"))
-                .filter(x -> x.contains("token=") || x.contains("s="))
-                .reduce("", String::concat);
-        FileLoader.updateCookie(cookie, website);
-    }
-
 }
diff --git a/src/main/resources/config.sys b/src/main/resources/config.sys
@@ -3,3 +3,8 @@
 
 ## 并行流底层线程数
 java.util.concurrent.ForkJoinPool.common.parallelism = 4
+
+## areaCode = 86 可选 默认国内
+## userID = 186xxxxxxxx
+## password = xxxx
+## rememberMe = true 可选 默认开启
diff --git a/src/test/java/mapperTest/StockToLongHuBangMapperTest.java b/src/test/java/mapperTest/StockToLongHuBangMapperTest.java
@@ -25,6 +25,7 @@ public class StockToLongHuBangMapperTest {
     public void testWrongArgument() throws Exception{
 
         Stock stock = new Stock("中飞股份", "SZ300489");
+        stock.setStockQueryDate(new Date());
         StockToLongHuBangMapper mapper = new StockToLongHuBangMapper();
         thrown.expect(IllegalArgumentException.class);
         thrown.expectMessage("lost parameter: stockQueryDate");