packagecom.wp.nevel.base.utils;importjava.io.BufferedOutputStream;importjava.io.BufferedReader;importjava.io.DataInputStream;importjava.io.DataOutputStream;importjava.io.File;importjava.io.FileInputStream;importjava.io.FileNotFoundException;importjava.io.FileOutputStream;importjava.io.FileReader;importjava.io.IOException;importjava.io.UnsupportedEncodingException;importjava.net.SocketException;importjava.net.UnknownHostException;importjava.util.ArrayList;importjava.util.List;importjava.util.Map;importjava.util.Map.Entry;importorg.apache.http.HttpEntity;importorg.apache.http.NameValuePair;importorg.apache.http.client.ClientPRotocolException;importorg.apache.http.client.HttpClient;importorg.apache.http.client.config.RequestConfig;importorg.apache.http.client.methods.CloseableHttpResponse;importorg.apache.http.client.methods.HttpGet;importorg.apache.http.client.utils.URLEncodedUtils;importorg.apache.http.impl.client.CloseableHttpClient;importorg.apache.http.impl.client.HttpClients;importorg.apache.http.message.BasicNameValuePair;importorg.apache.http.protocol.HttpContext;importorg.apache.http.util.EntityUtils;importorg.apache.log4j.Logger;importorg.junit.Test;importcom.wp.nevel.base.exception.ParserException;importcom.wp.nevel.base.service.impl.LogServiceHelp;publicclassHttpClientUtils {publicstaticLogger logger = Logger.getLogger( LogServiceHelp.class);privatestaticHttpClient httpclient;static{
httpclient=HttpClients.createDefault();
}
@Testpublicvoidtest(){
String url="http://www.shuchongw.com/files/article/html/23/23114/index.html";
doGetHtmlContent2byte(url);
}/*** 根据简单url获取网页数据,转换成byte [] 存储
**/publicstaticbyte[] doGetHtmlContent2byte(String url) {
CloseableHttpResponse response=null;byte[] resultByte ={};try{
HttpGet get=newHttpGet(url);
System.out.println(url);
RequestConfig requestConfig= RequestConfig.custom().setSocketTimeout(30000).setConnectTimeout(30000).build();
get.setConfig(requestConfig);try{
response=(CloseableHttpResponse) HttpClientUtils.httpclient.execute(get);
}catch(UnknownHostException e) {
e.printStackTrace();
logger.info("链接主网失败");
}intstatusCode =response.getStatusLine().getStatusCode();
System.out.println(statusCode);if(statusCode == 200) {
HttpEntity entity=response.getEntity();
resultByte=EntityUtils.toByteArray(entity);
}
}catch(ClientProtocolException e) {
e.printStackTrace();
}catch(SocketException e) {
e.printStackTrace();
}catch(IOException e){
e.printStackTrace();
}finally{try{if(response!=null){
response.close();
}
}catch(IOException e) {
e.printStackTrace();
}
}returnresultByte;
}/*** 根据复杂url获取网页数据,转换成byte [] 存储
*@throwsParserException
*@throwsIOException
*@throwsUnknownHostException
*@throwsClientProtocolException
*@throwsSocketException
**/publicstaticbyte[] doGetHtmlByParams2Byte(Map<String, String>params, String paramsEncoding, String url) {if(params !=null) {
List<NameValuePair> formparams =newArrayList<NameValuePair>();for(Entry<String, String>entry : params.entrySet()) {
formparams.add(newBasicNameValuePair(entry.getKey(), entry.getValue()));
}if(!formparams.isEmpty()) {
String paramsStr= URLEncodedUtils.format(formparams, paramsEncoding!=null?paramsEncoding:"utf-8");
url= url + "?" +paramsStr;
}
}returndoGetHtmlContent2byte(url);
}/*** 根据复杂url获取网页数据,转换成String 存储
*@throwsParserException
*@throwsIOException
*@throwsUnknownHostException
*@throwsClientProtocolException
*@throwsSocketException
**/publicstaticString doGetHtmlByParams2Text(Map<String, String> params, String paramsEncoding, String url,String htmlEncoding)throwsClientProtocolException, UnknownHostException, SocketException{try{returngetHtmlContentByText(doGetHtmlByParams2Byte(params,paramsEncoding,url),htmlEncoding!=null?htmlEncoding:"utf-8");
}catch(Exception e) {
e.printStackTrace();return"";
}
}/*** 根据简单url获取网页数据,转换成String 存储
*@throwsParserException
*@throwsIOException
*@throwsUnknownHostException
*@throwsClientProtocolException
*@throwsSocketException
**/publicstaticString doGetHtmlContentToString(String url, String encoding){try{returngetHtmlContentByText(doGetHtmlContent2byte(url),encoding);
}catch(Exception e) {
e.printStackTrace();return"";
}
}/*** 根据简单url获取网页图片数据, 保存路径[saveImagePath]
*@throwsParserException
*@throwsIOException
*@throwsUnknownHostException
*@throwsClientProtocolException
*@throwsSocketException
**/publicstaticvoidgetHtml2Image(String url,String saveImagPath){try{
downloadData(doGetHtmlContent2byte(url),saveImagPath);
}catch(Exception e) {
e.printStackTrace();
}
}publicstaticString getHtmlContentByText(byte[] htmlBytes, String encoding){try{returnnewString (htmlBytes,encoding!=null?encoding:"utf-8");
}catch(UnsupportedEncodingException e) {
e.printStackTrace();return"";
}
}/*** 执行下载io
*
*@parambyte [] data 网页字节流,filename 存储地址和文件名 return
**/publicstaticvoiddownloadData(byte[] data, String filename) {try{
DataOutputStream writer=newDataOutputStream(newFileOutputStream(newFile(filename)));
BufferedOutputStream out=newBufferedOutputStream(writer);
out.write(data);
out.flush();
out.close();
}catch(FileNotFoundException e) {
e.printStackTrace();
}catch(IOException e) {
e.printStackTrace();
}
}publicstaticString readFile(String filename)throwsIOException{
StringxmlContent=null;
File file=newFile(filename);
BufferedReader reader=null;try{if(!file.exists()){newRuntimeException("文件不存在");returnxmlContent;
}
StringBuilder buider=newStringBuilder();
String readata="";
reader=newBufferedReader(newFileReader(file));while(true){
readata=reader.readLine();if(readata==null){
break;
}
buider.append(readata).append("\n");
}
xmlContent=buider.toString();
}catch(FileNotFoundException e) {
e.printStackTrace();
}finally{if(reader!=null)
reader.close();
}returnxmlContent;
}publicstaticbyte[] doGetByteByHttpclient2Url(HttpContext httpContext,CloseableHttpClient client,String url){byte[] resultBytes =null;try{
HttpGet get=newHttpGet(url);
CloseableHttpResponse response=client.execute(get, httpContext);intstatus =response.getStatusLine().getStatusCode();
System.out.println("链接状态="+status);if(status!=200)returnresultBytes;
HttpEntity entity=response.getEntity();
resultBytes=EntityUtils.toByteArray(entity);
}catch(ClientProtocolException e) {thrownewRuntimeException("失败连接地址"+url, e);
}catch(IOException e) {thrownewRuntimeException("失败连接地址"+url, e);
}if(resultBytes==null){try{
Thread.sleep(2000);
}catch(InterruptedException e) {
e.printStackTrace();
}
}returnresultBytes;
}
}