php多线程抓取多个网页( php结合curl实现抓取我们再来看几个例子(1) )

　　php多线程抓取多个网页(

php结合curl实现抓取我们再来看几个例子(1)

)

　　PHP结合curl实现多线程爬取_PHP-php教程

　　PHP结合curl实现多线程爬取

　　让我们再看几个例子

　　(1)下面的代码是爬取多个url，然后将爬取到的url的页面代码写入到指定文件中

　　$urls = array('http://www.bitsCN.com/','http://www.google.com/','http://www.example.com/'); // 设置要抓取的页面 URL$save_to='/test.txt'; // 把抓取的代码写入该文件$st = fopen($save_to,"a");$mh = curl_multi_init();foreach ($urls as $i => $url) {$conn[$i] = curl_init($url);curl_setopt($conn[$i], CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)");curl_setopt($conn[$i], CURLOPT_HEADER ,0);curl_setopt($conn[$i], CURLOPT_CONNECTTIMEOUT,60);curl_setopt($conn[$i], CURLOPT_FILE,$st); // 将爬取的代码写入文件curl_multi_add_handle ($mh,$conn[$i]);} // 初始化do {curl_multi_exec($mh,$active);} while ($active); // 执行foreach ($urls as $i => $url) {curl_multi_remove_handle($mh,$conn[$i]);curl_close($conn[$i]);} // 结束清理curl_multi_close($mh);fclose($st);

　　(2)下面的代码和上面类似，只不过这个地方是先把获取到的代码放到一个变量中，然后再将获取到的内容写入到指定的文件中

　　$urls = array('http://www.bitsCN.com/','http://www.google.com/','http://www.example.com/');$save_to='/test.txt'; // 把抓取的代码写入该文件$st = fopen($save_to,"a");$mh = curl_multi_init();foreach ($urls as $i => $url) {$conn[$i] = curl_init($url);curl_setopt($conn[$i], CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)");curl_setopt($conn[$i], CURLOPT_HEADER ,0);curl_setopt($conn[$i], CURLOPT_CONNECTTIMEOUT,60);curl_setopt($conn[$i],CURLOPT_RETURNTRANSFER,true); // 不将爬取代码写到浏览器, 而是转化为字符串curl_multi_add_handle ($mh,$conn[$i]);}do {curl_multi_exec($mh,$active);} while ($active);foreach ($urls as $i => $url) {$data = curl_multi_getcontent($conn[$i]); // 获得爬取的代码字符串fwrite($st,$data); // 将字符串写入文件} // 获得数据变量, 并写入文件foreach ($urls as $i => $url) {curl_multi_remove_handle($mh,$conn[$i]);curl_close($conn[$i]);}curl_multi_close($mh);fclose($st);

　　(3)以下代码实现了使用PHP的Curl Functions实现文件的并发多线程下载

　　$urls=array( 'http://www.bitsCN.com/5w.zip', 'http://www.bitsCN.com/5w.zip', 'http://www.bitsCN.com/5w.zip');$save_to='./home/';$mh=curl_multi_init();foreach($urls as $i=>$url){ $g=$save_to.basename($url); if(!is_file($g)){ $conn[$i]=curl_init($url); $fp[$i]=fopen($g,"w"); curl_setopt($conn[$i],CURLOPT_USERAGENT,"Mozilla/4.0(compatible; MSIE 7.0; Windows NT 6.0)"); curl_setopt($conn[$i],CURLOPT_FILE,$fp[$i]); curl_setopt($conn[$i],CURLOPT_HEADER ,0); curl_setopt($conn[$i],CURLOPT_CONNECTTIMEOUT,60); curl_multi_add_handle($mh,$conn[$i]); }}do{ $n=curl_multi_exec($mh,$active);}while($active);foreach($urls as $i=>$url){ curl_multi_remove_handle($mh,$conn[$i]); curl_close($conn[$i]); fclose($fp[$i]);}curl_multi_close($mh);$urls=array( 'http://www.bitsCN.com/5w.zip', 'http://www.bitsCN.com/5w.zip', 'http://www.bitsCN.com/5w.zip');$save_to='./home/';$mh=curl_multi_init();foreach($urls as $i=>$url){ $g=$save_to.basename($url); if(!is_file($g)){ $conn[$i]=curl_init($url); $fp[$i]=fopen($g,"w"); curl_setopt($conn[$i],CURLOPT_USERAGENT,"Mozilla/4.0(compatible; MSIE 7.0; Windows NT 6.0)"); curl_setopt($conn[$i],CURLOPT_FILE,$fp[$i]); curl_setopt($conn[$i],CURLOPT_HEADER ,0); curl_setopt($conn[$i],CURLOPT_CONNECTTIMEOUT,60); curl_multi_add_handle($mh,$conn[$i]); }}do{ $n=curl_multi_exec($mh,$active);}while($active);foreach($urls as $i=>$url){ curl_multi_remove_handle($mh,$conn[$i]); curl_close($conn[$i]); fclose($fp[$i]);}curl_multi_close($mh);