技术文章 · 5360 阅读
kindle114下载中心电子书资源下载
上午刚刚弄好的， 结果下午就发现了， 好多资源没有办法下载，都需要k币的，测试了一下， 能免费下载的资源已经很少了。kindle114上资源还是挺不错的， 有很多电子书很适合kindle上阅读，电子书的资源很多， 根本看不过来， 最近朋友做微信推送电子书的功能。需要大量的电子书，正好，网上有源码用来下载这个站点，只是用的是python写的东西，好久没有了，python这东西，早就不知道丢哪去了。调试环境都没有， 直接用java弄一个好了。以下就是我做这个小工具的过程。
首先肯定是先看看网页的源文件，发现查找资源不需要登录， 但下载资源需要登录。没有登陆会跳转到错误页面。
本来一个很简单的东西，中间遇到的一个问题，一直怀疑是程序的问题，后来发现，还是不够仔细的问题。用多线程下载的时候， 始终下载的资源都是一样的，其实只是解析页面的时候，左边栏目中有一个热门下载没有排除掉。。看下面的这个图，每个页面都有，造成了干扰， 使用css过滤一下就行了。
还有一个很严重的错误， 就是没有使用jsoup获取页面源码的时候， 没有设置模拟浏览器终端的问题，导致了，总是访问页面，访问到手机端去了，这应该和服务器后台设置有关系， 默认成手机端页面， 会出现一个模板找不到的错误。
或者直接使用手机端返回的一个链接后缀也行， 如下图：
直接看源码：学java的人都能看懂， 涉及的jar包和资源文件，在这里下载 附件中： autorun1.bat:
java DLkindle114 swjsj.com jishijun f:\kindle114\ 1 2
rem 参数说明 参数中间空格隔开
rem swjsj.com为用户名 自己注册用户名
rem jishijun为密码 自己注册的密码
rem 'f:\kindle1141\' 文件路径
rem 1 开始下载页
rem 2 结束下载页
下载类源码
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class DLkindle114 {
public String username = "jishijun204";
public String password = "jishijun";
public String cookie_res = "";
public String file_path = "f:/kindle114/";
private int min_page = 1;
private int max_page = 1;
public static void main(String[] args) {
DLkindle114 dl = new DLkindle114();
if (args.length == 5) {
dl.username = args[0];
dl.password = args[1];
dl.file_path = args[2];
dl.min_page = Integer.parseInt(args[3]);
dl.max_page = Integer.parseInt(args[4]);
} else {
System.out.println("请输入参数,路径如：f:/kindle114/ 1 10表示路径为"
+ "f:/kindle114/，1第一页，10为最后一页 10页400条记录");
// return;
}
System.out.println(dl.min_page + "," + dl.max_page);
dl.downloadUrl();
dl.cookie_res = dl.getLoginCookie();
dl.downDZS();
}
private void downDZS() {
System.out.println(file_path + "dl_kindle114_" + min_page + "_"
+ max_page + ".log");
String str = getFileContent(file_path + "dl_kindle114_" + min_page
+ "_" + max_page + ".log");
String[] ss = str.split("\n");
for (String s : ss) {
downloadFile(s);
// new Thread(new Runnable() {
// public void run() {
// Test.downloadFile(s);
// }
// }).start();
}
}
private void downloadUrl() {
try {
for (int i = min_page; i <= max_page; i++) {
Connection con = Jsoup
.connect("http://www.kindle114.com/list-1-" + i
+ ".html");
con.header(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36"
+ " (KHTML, like Gecko) Chrome/37.0.2062.20 Safari/537.36");// 配置模拟浏览器
Document doc1 = con.get();
Elements links = doc1.select(".showatt_right .bm_c a[href]");
for (Element link : links) {
if (link.toString().indexOf("page") > -1) {
String absHref = link.attr("abs:href");
System.out.println(absHref);
Connection con1 = Jsoup.connect(absHref);
con1.header(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.1; WOW64) "
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
+ "Chrome/37.0.2062.20 Safari/537.36");// 配置模拟浏览器
// System.out.println(doc.html());
Document doc = con1.get();
Elements links1 = doc.select(".attbox_left a[href]");
for (Element link1 : links1) {
final String dl_link = link1.attr("abs:href");
if (dl_link.indexOf("mod=attachment") > -1) {
System.out.println(dl_link);
write(file_path, "dl_kindle114_" + min_page
+ "_" + max_page + ".log", dl_link
+ "\r\n", true, "UTF-8");
// new Thread(new Runnable() {
// public void run() {
// try {
// Test.downloadFile(dl_link);
// } catch (ConnectException e) {
// e.printStackTrace();
// }
// }
// }).start();
// try {
// Thread.currentThread().sleep(10000);
// } catch (InterruptedException e) {
// e.printStackTrace();
// }
}
}
}
}
}
} catch (IOException e) {
e.printStackTrace();
}
}
public void downloadFile(String url) {
if (url == null || url.equals(""))
return;
try {
URL u = new URL(url);
URLConnection c = u.openConnection();
c.setDoOutput(true);// 允许连接提交信息
c.setRequestProperty("method", "POST");// 网页提交方式“GET”、“POST”
c.setRequestProperty("Cookie", cookie_res);
String fileName = c.getHeaderField("Content-Disposition");
if (null == fileName) {
cookie_res = getLoginCookie();
c.setDoOutput(true);// 允许连接提交信息
c.setRequestProperty("method", "POST");// 网页提交方式“GET”、“POST”
c.setRequestProperty("Cookie", cookie_res);
fileName = c.getHeaderField("Content-Disposition");
}
fileName = new String(fileName.getBytes("ISO-8859-1"), "GBK");
c.connect();
InputStream is = c.getInputStream();
// Reader r = new InputStreamReader(is);
// BufferedReader br = new BufferedReader(r);
// String line = "";
// while ((line = br.readLine()) != null) {
// System.out.println(line);
// }
//
String filename = fileName.replaceAll("attachment; filename=", "")
.replaceAll("\"", "");
filename = filename.replaceAll("[\\-\\:：\\<>|\\?]", "_")
.replaceAll("\\s", "");
File d = new File(file_path);
if (!d.exists()) {
d.mkdirs();
}
String filezie = c.getHeaderField("Content-Length");
System.out.println("下载文件名称：" + fileName);
System.out.println("下载文件大小：" + Integer.parseInt(filezie) / 1024);
filename = d.getAbsolutePath() + "/" + filename;
OutputStream os = new FileOutputStream(filename);
int tmp = 0;
while ((tmp = is.read()) != -1) {
os.write(tmp);
}
os.close();
is.close();
} catch (IOException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}
}
public String getLoginCookie() {
String cookie_str = "";
CloseableHttpClient httpclient = HttpClients.createDefault();
try {
// Request.Post("http://targethost/login")
// .bodyForm(Form.form().add("username", "vip").add("password",
// "secret").build())
// .execute().returnContent();
HttpPost httpPost = new HttpPost(
"http://www.kindle114.com/member.php?mod=logging&action=login&loginsubmit=yes");
List nvps = new ArrayList();
nvps.add(new BasicNameValuePair("username", username));
nvps.add(new BasicNameValuePair("password", password));
httpPost.setEntity(new UrlEncodedFormEntity(nvps));
CloseableHttpResponse response2 = httpclient.execute(httpPost);
try {
// System.out.println(response2.getStatusLine());
HttpEntity entity2 = response2.getEntity();
Header[] cookies = response2.getHeaders("Set-Cookie");
for (Header c : cookies) {
// System.out.println("响应的参数cookie_name：" + c.getName());
String val = c.getValue();
val = val.substring(0, val.indexOf(";"));
// System.out.println("响应的参数cookie_value：" + val);
cookie_str = cookie_str.concat(val + ";");
}
// System.out.println("响应的参数cookie :" + cookie_str);
// InputStream is = entity2.getContent();
// Reader r = new InputStreamReader(is);
// BufferedReader br = new BufferedReader(r);
// String line = "";
// while((line = br.readLine()) != null){
// System.out.println(line);
// }
// do something useful with the response body
// and ensure it is fully consumed
EntityUtils.consume(entity2);
} finally {
response2.close();
}
} catch (ClientProtocolException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
try {
httpclient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return cookie_str;
}
/**
* 创建文件 写文件
*
* @param filePath
* 文件的所在路径
* @param fileName
* 文件名称
* @param fileCon
* 文件的内容
* @param blnAppend
* true 标识如果文件fileName在指定的路径下存在，则将文件打开并将写入的文件内容追加在文件的尾部 false
* 创建新的文件 如果文件存在则将文件覆盖
* @param charsetName
* 写文件的字符方式UTF-8或其它形式
* @return 如果文件创建失败 返回FALSE
*/
public boolean write(String filePath, String fileName, String fileCon,
boolean blnAppend, String charsetName) {
try {
if (filePath == null || filePath.trim().length() == 0) {
return false;
}
if (fileName == null || fileName.trim().length() == 0) {
return false;
}
if (fileCon == null || fileCon.trim().length() == 0) {
return false;
}
String strSep = System.getProperty("file.separator");
File file = new File(filePath);
if (!file.exists()) {
file.mkdirs();
}
String pathName = filePath + strSep + fileName;
FileOutputStream out = new FileOutputStream(pathName, blnAppend);
OutputStreamWriter writer;
if (null == charsetName || charsetName.trim().length() == 0) {
writer = new OutputStreamWriter(out);
} else {
writer = new OutputStreamWriter(out, charsetName);
}
writer.write(fileCon);
writer.flush();
writer.close();
return true;
} catch (Exception ex) {
return false;
}
}
public String getFileContent(String log_path) {
return OpenReadFile(new File(log_path), "UTF-8").toString();
}
/**
* 文件内容
*
* @param file
* 文件
* @param strCharset
* 文件字符集
* @return
*/
private StringBuffer OpenReadFile(File file, String strCharset) {
StringBuffer textCon = new StringBuffer();
// String strCharset = "";
try {
InputStreamReader reader = null;
if (strCharset.trim().length() == 0) {
reader = new InputStreamReader(new FileInputStream(file));
} else {
reader = new InputStreamReader(new FileInputStream(file),
Charset.forName(strCharset));
}
BufferedReader bufferedReader = new BufferedReader(reader);
String line = bufferedReader.readLine();
while (line != null) {
try {
String nextLine = bufferedReader.readLine();
if (nextLine != null) {
line += "\n";
}
textCon.append(line);
line = nextLine;
} catch (Exception blExc) {
blExc.printStackTrace();
}
}
reader.close();
bufferedReader.close();
} catch (IOException exc) {
exc.printStackTrace();
}
return textCon;
}
}