Java学习

java,spring,structs,hibernate,jsf,ireport,jfreechart,jasperreport,tomcat,jboss -----本博客已经搬家了,新的地址是 http://www.javaly.cn 如果有对文章有任何疑问或者有任何不懂的地方,欢迎到www.javaly.cn (Java乐园)指出,我会尽力帮助解决。一起进步

 

利用HttpClient获取网页内容

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
public class HttpClientExample {
// 获得ConnectionManager,设置相关参数
private static MultiThreadedHttpConnectionManager manager = new MultiThreadedHttpConnectionManager();
private static int connectionTimeOut = 20000;
private static int socketTimeOut = 10000;
private static int maxConnectionPerHost = 5;
private static int maxTotalConnections = 40;
// 标志初始化是否完成的flag
private static boolean initialed = false;
// 初始化ConnectionManger的方法
public static void SetPara() {
manager.getParams().setConnectionTimeout(connectionTimeOut);
manager.getParams().setSoTimeout(socketTimeOut);
manager.getParams().setDefaultMaxConnectionsPerHost(
maxConnectionPerHost);
manager.getParams().setMaxTotalConnections(maxTotalConnections);
initialed = true;
}
// 通过get方法获取网页内容
public static String getGetResponseWithHttpClient(String url, String encode) {
HttpClient client = new HttpClient(manager);
if (initialed) {
HttpClientExample.SetPara();
}
GetMethod get = new GetMethod(url);
get.setFollowRedirects(true);
String result = null;
StringBuffer resultBuffer = new StringBuffer();
try {
client.executeMethod(get);
// 在目标页面情况未知的条件下,不推荐使用getResponseBodyAsString()方法
// String strGetResponseBody = post.getResponseBodyAsString();
BufferedReader in = new BufferedReader(new InputStreamReader(get
.getResponseBodyAsStream(), get.getResponseCharSet()));
String inputLine = null;
while ((inputLine = in.readLine()) != null) {
resultBuffer.append(inputLine);
resultBuffer.append("\n");
}
in.close();
result = resultBuffer.toString();
// iso-8859-1 is the default reading encode
result = HttpClientExample.ConverterStringCode(resultBuffer
.toString(), get.getResponseCharSet(), encode);
} catch (Exception e) {
e.printStackTrace();
result = "";
} finally {
get.releaseConnection();
return result;
}
}
public static String getPostResponseWithHttpClient(String url, String encode) {
HttpClient client = new HttpClient(manager);
if (initialed) {
HttpClientExample.SetPara();
}
PostMethod post = new PostMethod(url);
post.setFollowRedirects(false);
StringBuffer resultBuffer = new StringBuffer();
String result = null;
try {
client.executeMethod(post);
BufferedReader in = new BufferedReader(new InputStreamReader(post
.getResponseBodyAsStream(), post.getResponseCharSet()));
String inputLine = null;
while ((inputLine = in.readLine()) != null) {
resultBuffer.append(inputLine);
resultBuffer.append("\n");
}
in.close();
// iso-8859-1 is the default reading encode
result = HttpClientExample.ConverterStringCode(resultBuffer
.toString(), post.getResponseCharSet(), encode);
} catch (Exception e) {
e.printStackTrace();
result = "";
} finally {
post.releaseConnection();
return result;
}
}
public static String getPostResponseWithHttpClient(String url,
String encode, NameValuePair[] nameValuePair) {
HttpClient client = new HttpClient(manager);
if (initialed) {
HttpClientExample.SetPara();
}
PostMethod post = new PostMethod(url);
post.setRequestBody(nameValuePair);
post.setFollowRedirects(false);
String result = null;
StringBuffer resultBuffer = new StringBuffer();
try {
client.executeMethod(post);
BufferedReader in = new BufferedReader(new InputStreamReader(post
.getResponseBodyAsStream(), post.getResponseCharSet()));
String inputLine = null;
while ((inputLine = in.readLine()) != null) {
resultBuffer.append(inputLine);
resultBuffer.append("\n");
}
in.close();
// iso-8859-1 is the default reading encode
result = HttpClientExample.ConverterStringCode(resultBuffer
.toString(), post.getResponseCharSet(), encode);
} catch (Exception e) {
e.printStackTrace();
result = "";
} finally {
post.releaseConnection();
return result;
}
}
private static String ConverterStringCode(String source, String srcEncode,
String destEncode) {
if (source != null) {
try {
return new String(source.getBytes(srcEncode), destEncode);
} catch (UnsupportedEncodingException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return "";
}
} else {
return "";
}
}
}


之后,就可以通过下面的代码获得目标网页:
String source = HttpClientExample.getGetResponseWithHttpClient("http://www.163.com", "GBK");

注意,在默认情况下,HttpClient的Request的Head中
User-Agent的值是Jakarta Commons-HttpClient 3.0RC1,如果需要改变它(例如,变为Mozilla/4.0),必须在调用之前运行如下语句:
System.getProperties().setProperty("httpclient.useragent", "Mozilla/4.0");
zhuan:http://www.diybl.com/course/3_program/java/javajs/20090210/155102.html

posted on 2009-04-28 09:28 找个美女做老婆 阅读(4497) 评论(0)  编辑  收藏


只有注册用户登录后才能发表评论。


网站导航:
 

导航

统计

公告

本blog已经搬到新家了, 新家:www.javaly.cn
 http://www.javaly.cn

常用链接

留言簿(6)

随笔档案

文章档案

搜索

最新评论

阅读排行榜

评论排行榜