<?php /** * 采集远程网址数据 */ class Get_Remote_data { /** * 从远程网址采集数据 * @param string $uri 远程地址 * @param string $rand_ip 随机ip * @param string $user_agent 模拟的user_agent * @param string $referer_uri 模拟的来路的地址 * @param array $post_data 需要post的数据,默认为空数组 * @param string $method 选择数据提交的数据,默认get * @return mixed */ public function get_data_from_remote($uri, $rand_ip, $user_agent, $referer_uri='', $post_data=array(), $method="get") { $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $uri); curl_setopt($ch, CURLOPT_TIMEOUT, 10); curl_setopt($ch, CURLOPT_HTTPHEADER, array('X-FORWARDED-FOR:'.$rand_ip, 'CLIENT-IP:'.$rand_ip)); //追踪返回302状态码,继续抓取 // curl_setopt($ch, CURLOPT_HEADER, true); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); //curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); if($method == 'post') { curl_setopt($ch, CURLOPT_POST, true); curl_setopt($ch, CURLOPT_POSTFIELDS, $post_data); } curl_setopt($ch, CURLOPT_NOBODY, false); if($method == 'get') { curl_setopt($ch, CURLOPT_REFERER, $referer_uri);//模拟来路 } curl_setopt($ch, CURLOPT_USERAGENT, $user_agent); $data = curl_exec($ch); # 获取数据 $http_code = curl_getinfo($ch,CURLINFO_HTTP_CODE); # 获取http状态码 curl_close($ch); return array( 'http_status' => $http_code, 'data' => $data ); } /** * 生成模拟来路地址 * @return string */ private function get_refer_uri() { $refer[0] = 'http://www.baidu.com'; $refer[1] = 'https://www.google.com'; $i = time() % count($refer); return $refer[$i]; } /** * 生成agent * @return string */ private function get_user_agent() { // windows + chrome $str[0] = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36"; // windows + ie $str[1] = "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; MASPJS; rv:11.0) like Gecko"; $i = time() % count($str); return $str[$i]; } /** * 生成随机ip * @return string */ private function get_rand_ip() { $arr_1 = array("218","218","66","66","218","218","60","60","202","204", "66","66","66","59","61","60","222","221","66","59","60", "60","66","218","218","62","63","64","66","66","122","211"); $randarr= mt_rand(0,count($arr_1)); $ip1id = $arr_1[$randarr]; $ip2id= round(rand(600000, 2550000) / 10000); $ip3id= round(rand(600000, 2550000) / 10000); $ip4id= round(rand(600000, 2550000) / 10000); return $ip1id . "." . $ip2id . "." . $ip3id . "." . $ip4id; } }
文章评论