Skip to content

Commit

Permalink
2.0md文件更新、默认分段数支持配置、更新相关代码
Browse files Browse the repository at this point in the history
  • Loading branch information
@monkeyWie
monkeyWie committed Jan 16, 2018
1 parent 0eea7f3 commit 16cd54c
Show file tree
Hide file tree
Showing 17 changed files with 372 additions and 145 deletions.
47 changes: 30 additions & 17 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,31 @@
### HTTP下载器
自动嗅探HTTP下载请求，支持HTTP、HTTPS，多连接分块下载，突破文件服务器单连接下载速度限制，基于[proxyee](https://github.com/monkeyWie/proxyee)实现。
#### HTTPS支持
浏览器访问http://serverIp:serverPort 进入证书下载页面，下载证书并安装(受信任的根证书颁发机构)
#### 内置插件
# HTTP下载器
使用本地http代理服务器方式嗅探下载请求，支持所有操作系统和浏览器(IE9+),支持分段下载和断点下载。
## 内置插件
1. 百度云大文件、合并下载限制突破
2. 百度云合并下载特殊优化
#### 自动重试
连接断开或超时自动重试
#### 效果预览
![百度云下载](https://raw.githubusercontent.com/monkeyWie/proxyee-down/dev/effect/bdy.gif)
### 运行环境
2. 百度云合并下载解压工具(可解压4G大文件)
## 使用教程
### 下载
[地址](https://github.com/monkeyWie/proxyee-down/releases)
*注：从1.5版本开始下载器仅支持64位操作系统，若是32位操作系统请下载1.5版本*。
### 安装
1. [Windows](https://github.com/monkeyWie/proxyee-down/blob/master/view/guide/windows.md)
2. [MAC](https://github.com/monkeyWie/proxyee-down/blob/master/view/guide/mac.md)
3. [Linux](https://github.com/monkeyWie/proxyee-down/blob/master/view/guide/linux.md)
### 常见问题
1. 浏览器地址栏显示红色的锁，并提示证书不安全？
*先重启浏览器，若还是有问题，按照安装教程中证书安装步骤重新安装一遍证书。*
2. 浏览器显示代理服务器无响应？
*先确认软件是否运行，再检查浏览器的代理设置是否正确。*
3. 百度云下载速度太慢？
*可能被百度云10kb限速了，请尝试下载文件夹或勾选多个文件一起下载。*
4. 百度云合并下载文件无法解压？
*可以使用下载器，工具栏的百度云解压工具进行解压。*

# 开发
本项目依赖[proxyee](https://github.com/monkeyWie/proxyee)，因为还没上传maven中央仓库，需自行编译打包至本地仓库。
## 环境
![](https://img.shields.io/badge/JAVA-1.8%2B-brightgreen.svg) ![](https://img.shields.io/badge/maven-3.0%2B-brightgreen.svg) ![](https://img.shields.io/badge/node.js-8.0%2B-brightgreen.svg)
#### 编译
## 编译
```
#proxyee依赖编译
git clone https://github.com/monkeyWie/proxyee.git
Expand All @@ -20,15 +34,14 @@ mvn clean install
#proxyee-down编译
git clone https://github.com/monkeyWie/proxyee-down.git
cd proxyee-down/view
cd proxyee-down/ui/view
npm install
npm run build
cd ..
mvn clean package
```
#### 运行
## 运行
```
cd target
//指定代理服务器端口9999
java -jar proxyee-down-1.0-SNAPSHOT.jar 9999
```
java -jar proxyee-down.jar
```
29 changes: 29 additions & 0 deletions common/src/main/java/lee/study/down/model/TaskInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import java.io.File;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
Expand Down Expand Up @@ -36,6 +37,34 @@ public String buildTaskRecordFilePath() {
return getFilePath() + File.separator + "." + getFileName() + ".inf";
}

public TaskInfo buildChunkInfoList(){
List<ChunkInfo> chunkInfoList = new ArrayList<>();
if (getTotalSize() > 0) { //非chunked编码
//计算chunk列表
for (int i = 0; i < getConnections(); i++) {
ChunkInfo chunkInfo = new ChunkInfo();
chunkInfo.setIndex(i);
long chunkSize = getTotalSize() / getConnections();
chunkInfo.setOriStartPosition(i * chunkSize);
chunkInfo.setNowStartPosition(chunkInfo.getOriStartPosition());
if (i == getConnections() - 1) { //最后一个连接去下载多出来的字节
chunkSize += getTotalSize() % getConnections();
}
chunkInfo.setEndPosition(chunkInfo.getOriStartPosition() + chunkSize - 1);
chunkInfo.setTotalSize(chunkSize);
chunkInfoList.add(chunkInfo);
}
} else { //chunked下载
ChunkInfo chunkInfo = new ChunkInfo();
chunkInfo.setIndex(0);
chunkInfo.setNowStartPosition(0);
chunkInfo.setOriStartPosition(0);
chunkInfoList.add(chunkInfo);
}
setChunkInfoList(chunkInfoList);
return this;
}

public void reset() {
startTime = lastTime = pauseTime = 0;
chunkInfoList.forEach((chunkInfo) -> {
Expand Down
6 changes: 4 additions & 2 deletions common/src/main/java/lee/study/down/util/PathUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,16 @@
import java.util.regex.Pattern;

public class PathUtil {

public static String ROOT_PATH;

static {
String path = Thread.currentThread().getContextClassLoader().getResource("").getPath();
Pattern pattern = Pattern.compile("^file:/(.*/)[^/]*\\.jar!/BOOT-INF/classes!/$");
Matcher matcher = pattern.matcher(path);
if(matcher.find()){
if (matcher.find()) {
ROOT_PATH = matcher.group(1);
}else {
} else {
ROOT_PATH = path;
}
}
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/lee/study/down/HttpDownBootstrap.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
import org.slf4j.LoggerFactory;

@Data
@AllArgsConstructor()
@AllArgsConstructor
public class HttpDownBootstrap {

private static final Logger LOGGER = LoggerFactory.getLogger(HttpDownBootstrap.class);
Expand Down
161 changes: 108 additions & 53 deletions core/src/main/java/lee/study/down/util/HttpDownUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,13 @@
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import lee.study.down.io.LargeMappedByteBuffer;
import lee.study.down.model.HttpRequestInfo;
import lee.study.down.model.TaskInfo;
import lee.study.proxyee.util.ProtoUtil;
import lee.study.proxyee.util.ProtoUtil.RequestProto;

public class HttpDownUtil {
Expand All @@ -38,63 +40,37 @@ public class HttpDownUtil {
* 检测是否支持断点下载
*/
public static TaskInfo getTaskInfo(HttpRequest httpRequest, HttpHeaders resHeaders,
SslContext clientSslCtx, NioEventLoopGroup loopGroup) throws InterruptedException {
SslContext clientSslCtx, NioEventLoopGroup loopGroup)
throws Exception {
HttpResponse httpResponse = null;
if (resHeaders == null) {
httpResponse = getResponse(httpRequest, clientSslCtx, loopGroup);
//处理重定向
if ((httpResponse.status().code() + "").indexOf("30") == 0) {
String redirectUrl = httpResponse.headers().get(HttpHeaderNames.LOCATION);
HttpRequestInfo requestInfo = (HttpRequestInfo) httpRequest;
requestInfo.headers().remove("Host");
requestInfo.setUri(redirectUrl);
RequestProto requestProto = ProtoUtil.getRequestProto(requestInfo);
requestInfo.headers().set("Host", requestProto.getHost());
requestInfo.setRequestProto(requestProto);
httpResponse = getResponse(httpRequest, clientSslCtx, loopGroup);
}
resHeaders = httpResponse.headers();
}
TaskInfo taskInfo = new TaskInfo()
.setId(UUID.randomUUID().toString())
.setFileName(getDownFileName(httpRequest, resHeaders))
.setTotalSize(getDownFileSize(resHeaders));
.setTotalSize(getDownFileTotalSize(resHeaders));
//chunked编码不支持断点下载
if (resHeaders.contains(HttpHeaderNames.CONTENT_LENGTH)) {
CountDownLatch cdl = new CountDownLatch(1);
HttpRequestInfo requestInfo = (HttpRequestInfo) httpRequest;
RequestProto requestProto = requestInfo.requestProto();
Bootstrap bootstrap = new Bootstrap();
bootstrap.group(loopGroup) // 注册线程池
.channel(NioSocketChannel.class) // 使用NioSocketChannel来作为连接用的channel类
.handler(new ChannelInitializer() {

@Override
protected void initChannel(Channel ch) throws Exception {
if (requestProto.getSsl()) {
ch.pipeline().addLast(clientSslCtx.newHandler(ch.alloc()));
}
ch.pipeline().addLast("httpCodec", new HttpClientCodec());
ch.pipeline().addLast(new ChannelInboundHandlerAdapter() {

@Override
public void channelRead(ChannelHandlerContext ctx0, Object msg0)
throws Exception {
if (msg0 instanceof HttpResponse) {
HttpResponse httpResponse = (HttpResponse) msg0;
//206表示支持断点下载
if (httpResponse.status().equals(HttpResponseStatus.PARTIAL_CONTENT)) {
taskInfo.setSupportRange(true);
}
ctx0.channel().close();
cdl.countDown();
}
}
});
}

});
ChannelFuture cf = bootstrap.connect(requestProto.getHost(), requestProto.getPort());
cf.addListener((ChannelFutureListener) future -> {
if (future.isSuccess()) {
//请求下载一个字节测试是否支持断点下载
httpRequest.headers().set(HttpHeaderNames.RANGE, "bytes=0-0");
cf.channel().writeAndFlush(httpRequest);
if (requestInfo.content() != null) {
//请求体写入
HttpContent content = new DefaultLastHttpContent();
content.content().writeBytes(requestInfo.content());
cf.channel().writeAndFlush(content);
}
} else {
cdl.countDown();
}
});
cdl.await(30, TimeUnit.SECONDS);
if (httpResponse == null) {
httpResponse = getResponse(httpRequest, clientSslCtx, loopGroup);
}
//206表示支持断点下载
if (httpResponse.status().equals(HttpResponseStatus.PARTIAL_CONTENT)) {
taskInfo.setSupportRange(true);
}
}
return taskInfo;
}
Expand Down Expand Up @@ -132,7 +108,7 @@ public static String getDownFileName(HttpRequest httpRequest, HttpHeaders resHea
}

/**
* 取下载文件的总大小
* 取当前请求下载文件的总大小
*/
public static long getDownFileSize(HttpHeaders resHeaders) {
String contentRange = resHeaders.get(HttpHeaderNames.CONTENT_RANGE);
Expand All @@ -153,6 +129,85 @@ public static long getDownFileSize(HttpHeaders resHeaders) {
return 0;
}

/**
* 取请求下载文件的总大小
*/
public static long getDownFileTotalSize(HttpHeaders resHeaders) {
String contentRange = resHeaders.get(HttpHeaderNames.CONTENT_RANGE);
if (contentRange != null) {
Pattern pattern = Pattern.compile("^.*/(\\d+).*$");
Matcher matcher = pattern.matcher(contentRange);
if (matcher.find()) {
return Long.parseLong(matcher.group(1));
}
} else {
String contentLength = resHeaders.get(HttpHeaderNames.CONTENT_LENGTH);
if (contentLength != null) {
return Long.valueOf(resHeaders.get(HttpHeaderNames.CONTENT_LENGTH));
}
}
return 0;
}

/**
* 取请求响应
*/
public static HttpResponse getResponse(HttpRequest httpRequest, SslContext clientSslCtx,
NioEventLoopGroup loopGroup) throws Exception {
final HttpResponse[] httpResponses = new HttpResponse[1];
CountDownLatch cdl = new CountDownLatch(1);
HttpRequestInfo requestInfo = (HttpRequestInfo) httpRequest;
RequestProto requestProto = requestInfo.requestProto();
Bootstrap bootstrap = new Bootstrap();
bootstrap.group(loopGroup) // 注册线程池
.channel(NioSocketChannel.class) // 使用NioSocketChannel来作为连接用的channel类
.handler(new ChannelInitializer() {

@Override
protected void initChannel(Channel ch) throws Exception {
if (requestProto.getSsl()) {
ch.pipeline().addLast(clientSslCtx.newHandler(ch.alloc()));
}
ch.pipeline().addLast("httpCodec", new HttpClientCodec());
ch.pipeline().addLast(new ChannelInboundHandlerAdapter() {

@Override
public void channelRead(ChannelHandlerContext ctx0, Object msg0)
throws Exception {
if (msg0 instanceof HttpResponse) {
HttpResponse httpResponse = (HttpResponse) msg0;
httpResponses[0] = httpResponse;
ctx0.channel().close();
cdl.countDown();
}
}
});
}

});
ChannelFuture cf = bootstrap.connect(requestProto.getHost(), requestProto.getPort());
cf.addListener((ChannelFutureListener) future -> {
if (future.isSuccess()) {
//请求下载一个字节测试是否支持断点下载
httpRequest.headers().set(HttpHeaderNames.RANGE, "bytes=0-0");
cf.channel().writeAndFlush(httpRequest);
if (requestInfo.content() != null) {
//请求体写入
HttpContent content = new DefaultLastHttpContent();
content.content().writeBytes(requestInfo.content());
cf.channel().writeAndFlush(content);
}
} else {
cdl.countDown();
}
});
cdl.await(30, TimeUnit.SECONDS);
if (httpResponses[0] == null) {
throw new TimeoutException("getResponse timeout");
}
return httpResponses[0];
}

public static void safeClose(Channel channel, FileChannel fileChannel,
LargeMappedByteBuffer mappedBuffer) throws IOException {
if (channel != null) {
Expand Down
4 changes: 1 addition & 3 deletions sniff/src/main/java/lee/study/down/HttpDownProxyServer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,7 @@ public void setProxyConfig(ProxyConfig proxyConfig) {

public void start(int port) {
LOGGER.debug("HttpDownProxyServer listen " + port + "\tproxyConfig:" + proxyConfig);
if (proxyConfig != null) {
proxyServer.proxyConfig(proxyConfig);
}
proxyServer.proxyConfig(proxyConfig);
//监听http下载请求
proxyServer.proxyInterceptInitializer(new HttpProxyInterceptInitializer() {
@Override
Expand Down
3 changes: 0 additions & 3 deletions ui/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,6 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>1.5.9.RELEASE</version>
<configuration>
<outputDirectory>../target</outputDirectory>
</configuration>
<executions>
<execution>
<goals>
Expand Down
6 changes: 4 additions & 2 deletions ui/src/main/java/lee/study/down/constant/HttpDownConstant.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
public class HttpDownConstant {

public static String HOME_PATH;
public static String LIB_PATH;
public static String TASK_RECORD_PATH;
public static String CONFIG_PATH;
public static SslContext clientSslContext;
Expand All @@ -29,8 +30,9 @@ public class HttpDownConstant {
} catch (UnsupportedEncodingException e) {
}
}
TASK_RECORD_PATH = HOME_PATH + File.separator + "records.inf";
CONFIG_PATH = HOME_PATH + File.separator + "config.inf";
LIB_PATH = HOME_PATH + "lib";
TASK_RECORD_PATH = HOME_PATH + "records.inf";
CONFIG_PATH = HOME_PATH + "config.inf";
try {
clientSslContext = SslContextBuilder.forClient()
.trustManager(InsecureTrustManagerFactory.INSTANCE)
Expand Down
Loading

0 comments on commit 16cd54c

Please sign in to comment.