由于所爬取的網(wǎng)站需要驗(yàn)證碼，通過網(wǎng)頁的開發(fā)人員工具【F12】及在線http post,get接口測(cè)試請(qǐng)求工具(http://coolaf.com/)發(fā)現(xiàn)訪問時(shí)加上請(qǐng)求頭header 信息時(shí)可以跳過驗(yàn)證碼校驗(yàn)。

而且該網(wǎng)站只接受post請(qǐng)求，對(duì)提交的參數(shù)也只接受json格式，否則請(qǐng)求失敗。

現(xiàn)將通過 post 方式提交json參數(shù)的方法記錄如下：

import java.io.UnsupportedEncodingException;import java.net.URI;import java.net.URLDecoder;import java.util.ArrayList;import java.util.List;import org.apache.http.HttpEntity;import org.apache.http.HttpResponse;import org.apache.http.client.HttpClient;import org.apache.http.client.config.RequestConfig;import org.apache.http.client.methods.HttpPost;import org.apache.http.client.methods.HttpRequestBase;import org.apache.http.client.utils.URIBuilder;import org.apache.http.entity.StringEntity;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClientBuilder;import org.apache.http.impl.client.HttpClients;import org.apache.http.util.EntityUtils;import com.alibaba.fastjson.JSONArray;import com.alibaba.fastjson.JSONObject;/** * <p>@PostJsonParamsTest.java</p> * @version 1.0 * @author zxk * @Date 2018-3-3 */public class PostJsonParamsTest { // 超時(shí)時(shí)間 private static final int RUN_TIME =10000; // 爬取初始頁數(shù) private String page; public static void main(String[] args) throws Exception { PostJsonParamsTest crawl = new PostJsonParamsTest(); // 請(qǐng)求的url地址 String url ='http://www.gzcredit.gov.cn/Service/CreditService.asmx/searchOrgWithPage'; // 設(shè)置起始訪問頁碼 crawl.setPage('1'); String isStop = ''; // 設(shè)置請(qǐng)求 HttpRequestBase request = null; request = new HttpPost(url); try { // 設(shè)置config RequestConfig requestConfig = RequestConfig.custom() .setSocketTimeout(RUN_TIME) .setConnectTimeout(RUN_TIME) .setConnectionRequestTimeout(RUN_TIME) .build(); request.setConfig(requestConfig); // json 格式的 post 參數(shù) String postParams ='{'condition':{'qymc':'%%%%','cydw':''},'pageNo':'+crawl.getPage()+','pageSize':100,count:2709846}'; System.out.println(postParams); HttpEntity httpEntity = new StringEntity(postParams); ((HttpPost) request).setEntity(httpEntity); // 添加請(qǐng)求頭，可以繞過驗(yàn)證碼 request.addHeader('Accept','application/json, text/javascript, */*'); request.addHeader('Accept-Encoding','gzip, deflate'); request.addHeader('Accept-Language', 'zh-CN,zh;q=0.8'); request.addHeader('Connection', 'keep-alive'); request.addHeader('Host', 'www.gzcredit.gov.cn'); request.addHeader('Content-Type', 'application/json; charset=UTF-8'); URIBuilder builder = new URIBuilder(url); URI uri = builder.build(); uri = new URI(URLDecoder.decode(uri.toString(), 'UTF-8')); request.setURI(uri); while(!isStop.equals('停止')||isStop.equals('重跑')){isStop = crawl.crawlList(request);if(isStop.equals('爬取')){ crawl.setPage(String.valueOf(Integer.parseInt(crawl.getPage())+1));}// if('2713'.equals(crawl.getPage())) break;if('2'.equals(crawl.getPage())){ break;} } } catch (NumberFormatException e) { e.printStackTrace(); throw new NumberFormatException('數(shù)字格式錯(cuò)誤'); } catch (UnsupportedEncodingException e) { e.printStackTrace(); throw new UnsupportedEncodingException('不支持的編碼集'); } } /** * 爬取搜索列表 * @param page * @return */ private String crawlList(HttpRequestBase request){ int statusCode = 0; // 下面兩種方式都可以用來創(chuàng)建客戶端連接,相當(dāng)于打開了一個(gè)瀏覽器 CloseableHttpClient httpClient = HttpClients.createDefault(); // HttpClient httpClient = HttpClientBuilder.create().build(); HttpEntity httpEntity = null; HttpResponse response = null; try { try {response = httpClient.execute(request); } catch (Exception e){e.printStackTrace();EntityUtils.consumeQuietly(httpEntity);return '重跑'; } //打印狀態(tài) statusCode =response.getStatusLine().getStatusCode(); if(statusCode!=200){EntityUtils.consumeQuietly(httpEntity);return '重跑'; } //實(shí)體 httpEntity = response.getEntity(); String searchListStr = EntityUtils.toString(httpEntity,'GBK').replaceAll('米', '米'); String allData = (String) JSONObject.parseObject(searchListStr).get('d'); // 字符串值中間含雙引號(hào)的替換處理 String s = allData.replaceAll('{'','{’') .replaceAll('':'', '’:’') .replaceAll('','', '’,’') .replaceAll('':', '’:') .replaceAll(','', ',’') .replaceAll(''}', '’}') .replaceAll(''', '') .replaceAll('’', ''') .replaceAll('<br />', '') .replaceAll('t', '') .replaceAll('', '?'); JSONObject jsonData = JSONObject.parseObject(s); JSONArray jsonContent = jsonData.getJSONArray('orgList'); searchListStr = null; allData = null; s = null; if (jsonContent==null || jsonContent.size()<1) {return '重跑'; } System.out.println(jsonContent.toJSONString()); return '爬取'; } catch (Exception e) { e.printStackTrace(); return '重跑'; } finally{ EntityUtils.consumeQuietly(httpEntity); } } private String getPage() { return page; } private void setPage(String page) { this.page = page; }}

補(bǔ)充知識(shí)：JAVA利用HttpClient發(fā)送post請(qǐng)求，將請(qǐng)求數(shù)據(jù)放到body里

我就廢話不多說了，大家還是直接看代碼吧~

/** * post請(qǐng)求 ，請(qǐng)求數(shù)據(jù)放到body里 * @param url 請(qǐng)求地址 * @param bodyData 參數(shù) * @author wangyj * @date 2019年4月20日 */ public static String doPostBodyData(String url, String bodyData) throws Exception{ String result = ''; CloseableHttpClient httpClient = null; CloseableHttpResponse response = null; try { HttpPost httpPost = getHttpPost(url, null); // 請(qǐng)求地址 httpPost.setEntity(new StringEntity(bodyData, Encoding)); httpClient = getHttpClient(); // 得到返回的response response = httpClient.execute(httpPost); HttpEntity entity = response.getEntity(); result = getResult(entity, Encoding); } catch (Exception e) { throw e; } finally { // 關(guān)閉httpClient if (null != httpClient) {httpClient.close(); } // 關(guān)閉response if (null != response) {EntityUtils.consume(response.getEntity()); // 會(huì)自動(dòng)釋放連接response.close(); } } return result; }

以上這篇java 實(shí)現(xiàn)通過 post 方式提交json參數(shù)操作就是小編分享給大家的全部內(nèi)容了，希望能給大家一個(gè)參考，也希望大家多多支持好吧啦網(wǎng)。