java爬虫抓取动态网页(【】动态cookie页面应该怎么设置才能抓取数据？ )

优采云发布时间: 2021-10-02 15:32

　　java爬虫抓取动态网页(【】动态cookie页面应该怎么设置才能抓取数据？

)

　　我需要抓取收录数据的网页内容，并使用httpurlconnection抓取网页内容（代码片段1）。但是，需要爬网的网页似乎在JS端设置了动态cookie，导致爬网的内容成为代码片段2。现在请帮助我了解如何设置这样的动态cookie页面来爬网数据？请给我发一封针对特定链接地址的私人信件~

/**

* 代码段1

*/

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.net.CookieHandler;

import java.net.CookieManager;

import java.net.CookiePolicy;

import java.net.HttpURLConnection;

import java.net.URL;

public class GetWebContent {

public static String GetWebContent(String urlString, final String charset,int timeout) throws IOException {

if (urlString == null || urlString.length() == 0) {

return null;

}

urlString = (urlString.startsWith("http://") || urlString

.startsWith("https://")) ? urlString : ("http://" + urlString)

.intern();

CookieHandler.setDefault(new CookieManager(null,

CookiePolicy.ACCEPT_ALL));

URL url = new URL(urlString);

HttpURLConnection conn = (HttpURLConnection) url.openConnection();

conn.setDoOutput(true);

conn.setRequestProperty("Pragma", "no-cache");

conn.setRequestProperty("Cache-Control", "no-cache");

// http://blog.csdn.net/yjflinchong

int temp = Integer.parseInt(Math.round(Math.random() * 2) + "");

conn.setRequestProperty("User-Agent", UserAgent[temp]); // 模拟手机系统

System.out.println(UserAgent[temp]);

conn.setRequestProperty("Accept",

"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");// 只接受text/html类型，当然也可以接受图片,pdf,*/*任意，就是tomcat/conf/web里面定义那些

conn.setConnectTimeout(timeout);

try {

if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) {

return null;

}

} catch (Exception e) {

try {

e.printStackTrace();

} catch (Exception e2) {

e2.printStackTrace();

}

return null;

}

InputStream input = conn.getInputStream();

BufferedReader reader = new BufferedReader(new InputStreamReader(input,

charset));

String line = null;

StringBuffer sb = new StringBuffer();

while ((line = reader.readLine()) != null) {

sb.append(line).append("\r\n");

}

if (reader != null) {

reader.close();

}

if (conn != null) {

conn.disconnect();

}

return sb.toString();

}

public static String[] UserAgent = {

//"Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.2",

"Mozilla/5.0 (iPad; U; CPU OS 9_2_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B500 Safari/531.21.11",

// http://blog.csdn.net/yjflinchong

"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36",//win10 chrome

"Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1" };//iphone X

}

　　代码段2，抓取从网页获取的数据

var data = {

'cookie' : "6a947161cfdda35d01f8c6c673653ffc",

'uri' : "此处地址即为需要抓取的地址，如有需要请私信我"

};

function getCookie(c_name){

if (document.cookie.length>0){

c_start=document.cookie.indexOf(c_name + "=")

if (c_start!=-1){

c_start=c_start + c_name.length+1

c_end=document.cookie.indexOf(";",c_start)

if (c_end==-1) c_end=document.cookie.length

return unescape(document.cookie.substring(c_start,c_end))

}

}

return "";

}

function setCookie(c_name,value,expiredays){

var exdate=new Date();

exdate.setDate(exdate.getDate()+expiredays);

document.cookie=c_name+ "=" +escape(value)+((expiredays==null) ? "" : "; path=/; expires="+exdate.toGMTString());

}

function jump(){

setCookie('elvaeye',data['cookie'],365);

window.location = data['uri'];

}

0

2021-10-02

java爬虫抓取动态网页

0 个评论

要回复文章请先登录或注册

AI时代内容工厂

java爬虫抓取动态网页(【】动态cookie页面应该怎么设置才能抓取数据？ )

0 个评论

发起人