使用java實現http多線程斷點下載文件(一)

2019-11-26 16:16:36

字體：大中小

來源：轉載

供稿：網友

基本原理：利用URLConnection獲取要下載文件的長度、頭部等相關信息，并設置響應的頭部信息。并且通過URLConnection獲取輸入流，將文件分成指定的塊，每一塊單獨開辟一個線程完成數據的讀取、寫入。通過輸入流讀取下載文件的信息，然后將讀取的信息用RandomAccessFile隨機寫入到本地文件中。同時，每個線程寫入的數據都文件指針也就是寫入數據的長度，需要保存在一個臨時文件中。這樣當本次下載沒有完成的時候，下次下載的時候就從這個文件中讀取上一次下載的文件長度，然后繼續接著上一次的位置開始下載。并且將本次下載的長度寫入到這個文件中。

一、下載文件信息類、實體
封裝即將下載資源的信息

復制代碼代碼如下:

 
package com.hoo.entity; 
/** 
* <b>function:</b> 下載文件信息類 
* @author hoojo 
* @createDate 2011-9-21 下午05:14:58 
* @file DownloadInfo.java 
* @package com.hoo.entity 
* @project MultiThreadDownLoad 
* @blog http://blog.csdn.net/IBM_hoojo 
* @email hoojo_@126.com 
* @version 1.0 
*/ 
public class DownloadInfo { 
//下載文件url 
private String url; 
//下載文件名稱 
private String fileName; 
//下載文件路徑 
private String filePath; 
//分成多少段下載， 每一段用一個線程完成下載 
private int splitter; 
//下載文件默認保存路徑 
private final static String FILE_PATH = "C:/temp"; 
//默認分塊數、線程數 
private final static int SPLITTER_NUM = 5; 
public DownloadInfo() { 
super(); 
} 
/** 
* @param url 下載地址 
*/ 
public DownloadInfo(String url) { 
this(url, null, null, SPLITTER_NUM); 
} 
/** 
* @param url 下載地址url 
* @param splitter 分成多少段或是多少個線程下載 
*/ 
public DownloadInfo(String url, int splitter) { 
this(url, null, null, splitter); 
} 
/*** 
* @param url 下載地址 
* @param fileName 文件名稱 
* @param filePath 文件保存路徑 
* @param splitter 分成多少段或是多少個線程下載 
*/ 
public DownloadInfo(String url, String fileName, String filePath, int splitter) { 
super(); 
if (url == null || "".equals(url)) { 
throw new RuntimeException("url is not null!"); 
} 
this.url = url; 
this.fileName = (fileName == null || "".equals(fileName)) ? getFileName(url) : fileName; 
this.filePath = (filePath == null || "".equals(filePath)) ? FILE_PATH : filePath; 
this.splitter = (splitter < 1) ? SPLITTER_NUM : splitter; 
} 
/** 
* <b>function:</b> 通過url獲得文件名稱 
* @author hoojo 
* @createDate 2011-9-30 下午05:00:00 
* @param url 
* @return 
*/ 
private String getFileName(String url) { 
return url.substring(url.lastIndexOf("/") + 1, url.length()); 
} 
public String getUrl() { 
return url; 
} 
public void setUrl(String url) { 
if (url == null || "".equals(url)) { 
throw new RuntimeException("url is not null!"); 
} 
this.url = url; 
} 
public String getFileName() { 
return fileName; 
} 
public void setFileName(String fileName) { 
this.fileName = (fileName == null || "".equals(fileName)) ? getFileName(url) : fileName; 
} 
public String getFilePath() { 
return filePath; 
} 
public void setFilePath(String filePath) { 
this.filePath = (filePath == null || "".equals(filePath)) ? FILE_PATH : filePath; 
} 
public int getSplitter() { 
return splitter; 
} 
public void setSplitter(int splitter) { 
this.splitter = (splitter < 1) ? SPLITTER_NUM : splitter; 
} 
@Override 
public String toString() { 
return this.url + "#" + this.fileName + "#" + this.filePath + "#" + this.splitter; 
} 
} 

二、隨機寫入一段文件

復制代碼代碼如下:

 
package com.hoo.download; 
import java.io.IOException; 
import java.io.RandomAccessFile; 
/** 
* <b>function:</b> 寫入文件、保存文件 
* @author hoojo 
* @createDate 2011-9-21 下午05:44:02 
* @file SaveItemFile.java 
* @package com.hoo.download 
* @project MultiThreadDownLoad 
* @blog http://blog.csdn.net/IBM_hoojo 
* @email hoojo_@126.com 
* @version 1.0 
*/ 
public class SaveItemFile { 
//存儲文件 
private RandomAccessFile itemFile; 
public SaveItemFile() throws IOException { 
this("", 0); 
} 
/** 
* @param name 文件路徑、名稱 
* @param pos 寫入點位置 position 
* @throws IOException 
*/ 
public SaveItemFile(String name, long pos) throws IOException { 
itemFile = new RandomAccessFile(name, "rw"); 
//在指定的pos位置開始寫入數據 
itemFile.seek(pos); 
} 
/** 
* <b>function:</b> 同步方法寫入文件 
* @author hoojo 
* @createDate 2011-9-26 下午12:21:22 
* @param buff 緩沖數組 
* @param start 起始位置 
* @param length 長度 
* @return 
*/ 
public synchronized int write(byte[] buff, int start, int length) { 
int i = -1; 
try { 
itemFile.write(buff, start, length); 
i = length; 
} catch (IOException e) { 
e.printStackTrace(); 
} 
return i; 
} 
public void close() throws IOException { 
if (itemFile != null) { 
itemFile.close(); 
} 
} 
} 

這個類主要是完成向本地的指定文件指針出開始寫入文件，并返回當前寫入文件的長度（文件指針）。這個類將被線程調用，文件被分成對應的塊后，將被線程調用。每個線程都將會調用這個類完成文件的隨機寫入。

三、單個線程下載文件

復制代碼代碼如下:

 
package com.hoo.download; 
import java.io.IOException; 
import java.io.InputStream; 
import java.net.HttpURLConnection; 
import java.net.MalformedURLException; 
import java.net.URL; 
import java.net.URLConnection; 
import com.hoo.util.LogUtils; 
/** 
* <b>function:</b> 單線程下載文件 
* @author hoojo 
* @createDate 2011-9-22 下午02:55:10 
* @file DownloadFile.java 
* @package com.hoo.download 
* @project MultiThreadDownLoad 
* @blog http://blog.csdn.net/IBM_hoojo 
* @email hoojo_@126.com 
* @version 1.0 
*/ 
public class DownloadFile extends Thread { 
//下載文件url 
private String url; 
//下載文件起始位置 
private long startPos; 
//下載文件結束位置 
private long endPos; 
//線程id 
private int threadId; 
//下載是否完成 
private boolean isDownloadOver = false; 
private SaveItemFile itemFile; 
private static final int BUFF_LENGTH = 1024 * 8; 
/** 
* @param url 下載文件url 
* @param name 文件名稱 
* @param startPos 下載文件起點 
* @param endPos 下載文件結束點 
* @param threadId 線程id 
* @throws IOException 
*/ 
public DownloadFile(String url, String name, long startPos, long endPos, int threadId) throws IOException { 
super(); 
this.url = url; 
this.startPos = startPos; 
this.endPos = endPos; 
this.threadId = threadId; 
//分塊下載寫入文件內容 
this.itemFile = new SaveItemFile(name, startPos); 
} 
@Override 
public void run() { 
while (endPos > startPos && !isDownloadOver) { 
try { 
URL url = new URL(this.url); 
HttpURLConnection conn = (HttpURLConnection) url.openConnection(); 
// 設置連接超時時間為10000ms 
conn.setConnectTimeout(10000); 
// 設置讀取數據超時時間為10000ms 
conn.setReadTimeout(10000); 
setHeader(conn); 
String property = "bytes=" + startPos + "-"; 
conn.setRequestProperty("RANGE", property); 
//輸出log信息 
LogUtils.log("開始 " + threadId + "：" + property + endPos); 
//printHeader(conn); 
//獲取文件輸入流，讀取文件內容 
InputStream is = conn.getInputStream(); 
byte[] buff = new byte[BUFF_LENGTH]; 
int length = -1; 
LogUtils.log("#start#Thread: " + threadId + ", startPos: " + startPos + ", endPos: " + endPos); 
while ((length = is.read(buff)) > 0 && startPos < endPos && !isDownloadOver) { 
//寫入文件內容，返回最后寫入的長度 
startPos += itemFile.write(buff, 0, length); 
} 
LogUtils.log("#over#Thread: " + threadId + ", startPos: " + startPos + ", endPos: " + endPos); 
LogUtils.log("Thread " + threadId + " is execute over!"); 
this.isDownloadOver = true; 
} catch (MalformedURLException e) { 
e.printStackTrace(); 
} catch (IOException e) { 
e.printStackTrace(); 
} finally { 
try { 
if (itemFile != null) { 
itemFile.close(); 
} 
} catch (IOException e) { 
e.printStackTrace(); 
} 
} 
} 
if (endPos < startPos && !isDownloadOver) { 
LogUtils.log("Thread " + threadId + " startPos > endPos, not need download file !"); 
this.isDownloadOver = true; 
} 
if (endPos == startPos && !isDownloadOver) { 
LogUtils.log("Thread " + threadId + " startPos = endPos, not need download file !"); 
this.isDownloadOver = true; 
} 
} 
/** 
* <b>function:</b> 打印下載文件頭部信息 
* @author hoojo 
* @createDate 2011-9-22 下午05:44:35 
* @param conn HttpURLConnection 
*/ 
public static void printHeader(URLConnection conn) { 
int i = 1; 
while (true) { 
String header = conn.getHeaderFieldKey(i); 
i++; 
if (header != null) { 
LogUtils.info(header + ":" + conn.getHeaderField(i)); 
} else { 
break; 
} 
} 
} 
/** 
* <b>function:</b> 設置URLConnection的頭部信息，偽裝請求信息 
* @author hoojo 
* @createDate 2011-9-28 下午05:29:43 
* @param con 
*/ 
public static void setHeader(URLConnection conn) { 
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3"); 
conn.setRequestProperty("Accept-Language", "en-us,en;q=0.7,zh-cn;q=0.3"); 
conn.setRequestProperty("Accept-Encoding", "utf-8"); 
conn.setRequestProperty("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7"); 
conn.setRequestProperty("Keep-Alive", "300"); 
conn.setRequestProperty("connnection", "keep-alive"); 
conn.setRequestProperty("If-Modified-Since", "Fri, 02 Jan 2009 17:00:05 GMT"); 
conn.setRequestProperty("If-None-Match", "/"1261d8-4290-df64d224/""); 
conn.setRequestProperty("Cache-conntrol", "max-age=0"); 
conn.setRequestProperty("Referer", "http://www.baidu.com"); 
} 
public boolean isDownloadOver() { 
return isDownloadOver; 
} 
public long getStartPos() { 
return startPos; 
} 
public long getEndPos() { 
return endPos; 
} 
} 

這個類主要是完成單個線程的文件下載，將通過URLConnection讀取指定url的資源信息。然后用InputStream讀取文件內容，然后調用調用SaveItemFile類，向本地寫入當前要讀取的塊的內容。

四、分段多線程寫入文件內容

復制代碼代碼如下:

 
package com.hoo.download; 
import java.io.DataInputStream; 
import java.io.DataOutputStream; 
import java.io.File; 
import java.io.FileInputStream; 
import java.io.FileOutputStream; 
import java.io.IOException; 
import java.net.HttpURLConnection; 
import java.net.MalformedURLException; 
import java.net.URL; 
import com.hoo.entity.DownloadInfo; 
import com.hoo.util.LogUtils; 
/** 
* <b>function:</b> 分批量下載文件 
* @author hoojo 
* @createDate 2011-9-22 下午05:51:54 
* @file BatchDownloadFile.java 
* @package com.hoo.download 
* @project MultiThreadDownLoad 
* @blog http://blog.csdn.net/IBM_hoojo 
* @email hoojo_@126.com 
* @version 1.0 
*/ 
public class BatchDownloadFile implements Runnable { 
//下載文件信息 
private DownloadInfo downloadInfo; 
//一組開始下載位置 
private long[] startPos; 
//一組結束下載位置 
private long[] endPos; 
//休眠時間 
private static final int SLEEP_SECONDS = 500; 
//子線程下載 
private DownloadFile[] fileItem; 
//文件長度 
private int length; 
//是否第一個文件 
private boolean first = true; 
//是否停止下載 
private boolean stop = false; 
//臨時文件信息 
private File tempFile; 
public BatchDownloadFile(DownloadInfo downloadInfo) { 
this.downloadInfo = downloadInfo; 
String tempPath = this.downloadInfo.getFilePath() + File.separator + downloadInfo.getFileName() + ".position"; 
tempFile = new File(tempPath); 
//如果存在讀入點位置的文件 
if (tempFile.exists()) { 
first = false; 
//就直接讀取內容 
try { 
readPosInfo(); 
} catch (IOException e) { 
e.printStackTrace(); 
} 
} else { 
//數組的長度就要分成多少段的數量 
startPos = new long[downloadInfo.getSplitter()]; 
endPos = new long[downloadInfo.getSplitter()]; 
} 
} 
@Override 
public void run() { 
//首次下載，獲取下載文件長度 
if (first) { 
length = this.getFileSize();//獲取文件長度 
if (length == -1) { 
LogUtils.log("file length is know!"); 
stop = true; 
} else if (length == -2) { 
LogUtils.log("read file length is error!"); 
stop = true; 
} else if (length > 0) { 
/** 
* eg 
* start: 1, 3, 5, 7, 9 
* end: 3, 5, 7, 9, length 
*/ 
for (int i = 0, len = startPos.length; i < len; i++) { 
int size = i * (length / len); 
startPos[i] = size; 
//設置最后一個結束點的位置 
if (i == len - 1) { 
endPos[i] = length; 
} else { 
size = (i + 1) * (length / len); 
endPos[i] = size; 
} 
LogUtils.log("start-end Position[" + i + "]: " + startPos[i] + "-" + endPos[i]); 
} 
} else { 
LogUtils.log("get file length is error, download is stop!"); 
stop = true; 
} 
} 
//子線程開始下載 
if (!stop) { 
//創建單線程下載對象數組 
fileItem = new DownloadFile[startPos.length];//startPos.length = downloadInfo.getSplitter() 
for (int i = 0; i < startPos.length; i++) { 
try { 
//創建指定個數單線程下載對象，每個線程獨立完成指定塊內容的下載 
fileItem[i] = new DownloadFile( 
downloadInfo.getUrl(), 
this.downloadInfo.getFilePath() + File.separator + downloadInfo.getFileName(), 
startPos[i], endPos[i], i 
); 
fileItem[i].start();//啟動線程，開始下載 
LogUtils.log("Thread: " + i + ", startPos: " + startPos[i] + ", endPos: " + endPos[i]); 
} catch (IOException e) { 
e.printStackTrace(); 
} 
} 
//循環寫入下載文件長度信息 
while (!stop) { 
try { 
writePosInfo(); 
LogUtils.log("downloading……"); 
Thread.sleep(SLEEP_SECONDS); 
stop = true; 
} catch (IOException e) { 
e.printStackTrace(); 
} catch (InterruptedException e) { 
e.printStackTrace(); 
} 
for (int i = 0; i < startPos.length; i++) { 
if (!fileItem[i].isDownloadOver()) { 
stop = false; 
break; 
} 
} 
} 
LogUtils.info("Download task is finished!"); 
} 
} 
/** 
* 將寫入點數據保存在臨時文件中 
* @author hoojo 
* @createDate 2011-9-23 下午05:25:37 
* @throws IOException 
*/ 
private void writePosInfo() throws IOException { 
DataOutputStream dos = new DataOutputStream(new FileOutputStream(tempFile)); 
dos.writeInt(startPos.length); 
for (int i = 0; i < startPos.length; i++) { 
dos.writeLong(fileItem[i].getStartPos()); 
dos.writeLong(fileItem[i].getEndPos()); 
//LogUtils.info("[" + fileItem[i].getStartPos() + "#" + fileItem[i].getEndPos() + "]"); 
} 
dos.close(); 
} 
/** 
* <b>function:</b>讀取寫入點的位置信息 
* @author hoojo 
* @createDate 2011-9-23 下午05:30:29 
* @throws IOException 
*/ 
private void readPosInfo() throws IOException { 
DataInputStream dis = new DataInputStream(new FileInputStream(tempFile)); 
int startPosLength = dis.readInt(); 
startPos = new long[startPosLength]; 
endPos = new long[startPosLength]; 
for (int i = 0; i < startPosLength; i++) { 
startPos[i] = dis.readLong(); 
endPos[i] = dis.readLong(); 
} 
dis.close(); 
} 
/** 
* <b>function:</b> 獲取下載文件的長度 
* @author hoojo 
* @createDate 2011-9-26 下午12:15:08 
* @return 
*/ 
private int getFileSize() { 
int fileLength = -1; 
try { 
URL url = new URL(this.downloadInfo.getUrl()); 
HttpURLConnection conn = (HttpURLConnection) url.openConnection(); 
DownloadFile.setHeader(conn); 
int stateCode = conn.getResponseCode(); 
//判斷http status是否為HTTP/1.1 206 Partial Content或者200 OK 
if (stateCode != HttpURLConnection.HTTP_OK && stateCode != HttpURLConnection.HTTP_PARTIAL) { 
LogUtils.log("Error Code: " + stateCode); 
return -2; 
} else if (stateCode >= 400) { 
LogUtils.log("Error Code: " + stateCode); 
return -2; 
} else { 
//獲取長度 
fileLength = conn.getContentLength(); 
LogUtils.log("FileLength: " + fileLength); 
} 
//讀取文件長度 
/*for (int i = 1; ; i++) { 
String header = conn.getHeaderFieldKey(i); 
if (header != null) { 
if ("Content-Length".equals(header)) { 
fileLength = Integer.parseInt(conn.getHeaderField(i)); 
break; 
} 
} else { 
break; 
} 
} 
*/ 
DownloadFile.printHeader(conn); 
} catch (MalformedURLException e) { 
e.printStackTrace(); 
} catch (IOException e) { 
e.printStackTrace(); 
} 
return fileLength; 
} 
} 

這個類主要是完成讀取指定url資源的內容，獲取該資源的長度。然后將該資源分成指定的塊數，將每塊的起始下載位置、結束下載位置，分別保存在一個數組中。每塊都單獨開辟一個獨立線程開始下載。在開始下載之前，需要創建一個臨時文件，寫入當前下載線程的開始下載指針位置和結束下載指針位置。

五、工具類、測試類
日志工具類

復制代碼代碼如下:

 
package com.hoo.util; 
/** 
* <b>function:</b> 日志工具類 
* @author hoojo 
* @createDate 2011-9-21 下午05:21:27 
* @file LogUtils.java 
* @package com.hoo.util 
* @project MultiThreadDownLoad 
* @blog http://blog.csdn.net/IBM_hoojo 
* @email hoojo_@126.com 
* @version 1.0 
*/ 
public abstract class LogUtils { 
public static void log(Object message) { 
System.err.println(message); 
} 
public static void log(String message) { 
System.err.println(message); 
} 
public static void log(int message) { 
System.err.println(message); 
} 
public static void info(Object message) { 
System.out.println(message); 
} 
public static void info(String message) { 
System.out.println(message); 
} 
public static void info(int message) { 
System.out.println(message); 
} 
} 

下載工具類

復制代碼代碼如下:

 
package com.hoo.util; 
import com.hoo.download.BatchDownloadFile; 
import com.hoo.entity.DownloadInfo; 
/** 
* <b>function:</b> 分塊多線程下載工具類 
* @author hoojo 
* @createDate 2011-9-28 下午05:22:18 
* @file DownloadUtils.java 
* @package com.hoo.util 
* @project MultiThreadDownLoad 
* @blog http://blog.csdn.net/IBM_hoojo 
* @email hoojo_@126.com 
* @version 1.0 
*/ 
public abstract class DownloadUtils { 
public static void download(String url) { 
DownloadInfo bean = new DownloadInfo(url); 
LogUtils.info(bean); 
BatchDownloadFile down = new BatchDownloadFile(bean); 
new Thread(down).start(); 
} 
public static void download(String url, int threadNum) { 
DownloadInfo bean = new DownloadInfo(url, threadNum); 
LogUtils.info(bean); 
BatchDownloadFile down = new BatchDownloadFile(bean); 
new Thread(down).start(); 
} 
public static void download(String url, String fileName, String filePath, int threadNum) { 
DownloadInfo bean = new DownloadInfo(url, fileName, filePath, threadNum); 
LogUtils.info(bean); 
BatchDownloadFile down = new BatchDownloadFile(bean); 
new Thread(down).start(); 
} 
} 

下載測試類

復制代碼代碼如下:

 
package com.hoo.test; 
import com.hoo.util.DownloadUtils; 
/** 
* <b>function:</b> 下載測試 
* @author hoojo 
* @createDate 2011-9-23 下午05:49:46 
* @file TestDownloadMain.java 
* @package com.hoo.download 
* @project MultiThreadDownLoad 
* @blog http://blog.csdn.net/IBM_hoojo 
* @email hoojo_@126.com 
* @version 1.0 
*/ 
public class TestDownloadMain { 
public static void main(String[] args) { 
/*DownloadInfo bean = new DownloadInfo("http://i7.meishichina.com/Health/UploadFiles/201109/2011092116224363.jpg"); 
System.out.println(bean); 
BatchDownloadFile down = new BatchDownloadFile(bean); 
new Thread(down).start();*/ 
//DownloadUtils.download("http://i7.meishichina.com/Health/UploadFiles/201109/2011092116224363.jpg"); 
DownloadUtils.download("http://mp3.baidu.com/j?j=2&url=http%3A%2F%2Fzhangmenshiting2.baidu.com%2Fdata%2Fmusic%2F1669425%2F%25E9%2599%25B7%25E5%2585%25A5%25E7%2588%25B1%25E9%2587%258C%25E9%259D%25A2.mp3%3Fxcode%3D2ff36fb70737c816553396c56deab3f1", "aa.mp3", "c:/temp", 5); 
} 
} 