本文目录一览：
php中curl爬虫 怎么样通过网页获取所有链接
本文承接上面两篇，本篇中的示例要调用到前两篇中的函数，做一个简单的URL采集。一般php采集网络数据会用file_get_contents、file和cURL。不过据说cURL会比file_get_contents、file更快更专业，更适合采集。今天就试试用cURL来获取网页上的所有链接。示例如下：
?php
/*
* 使用curl 采集hao123.com下的所有链接。
*/
include_once('function.php');
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, '');
// 只需返回HTTP header
curl_setopt($ch, CURLOPT_HEADER, 1);
// 页面内容我们并不需要
// curl_setopt($ch, CURLOPT_NOBODY, 1);
// 返回结果，而不是输出它
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
$html = curl_exec($ch);
$info = curl_getinfo($ch);
if ($html === false) {
echo "cURL Error: " . curl_error($ch);
}
curl_close($ch);
$linkarr = _striplinks($html);
// 主机部分，补全用
$host = '';
if (is_array($linkarr)) {
foreach ($linkarr as $k = $v) {
$linkresult[$k] = _expandlinks($v, $host);
}
}
printf("p此页面的所有链接为：/ppre%s/pren", var_export($linkresult , true));
?
function.php内容如下（即为上两篇中两个函数的合集）：
?php
function _striplinks($document) {
preg_match_all("'s*as.*?hrefs*=s*(["'])?(?(1) (.*?)\1 | ([^s]+))'isx", $document, $links);
// catenate the non-empty matches from the conditional subpattern
while (list($key, $val) = each($links[2])) {
if (!empty($val))
$match[] = $val;
} while (list($key, $val) = each($links[3])) {
if (!empty($val))
$match[] = $val;
}
// return the links
return $match;
}
/*===================================================================*
Function: _expandlinks
Purpose: expand each link into a fully qualified URL
Input: $links the links to qualify
$URI the full URI to get the base from
Output: $expandedLinks the expanded links
*===================================================================*/
function _expandlinks($links,$URI)
{
$URI_PARTS = parse_url($URI);
$host = $URI_PARTS["host"];
preg_match("/^[^?]+/",$URI,$match);
$match = preg_replace("|/[^/.]+.[^/.]+$|","",$match[0]);
$match = preg_replace("|/$|","",$match);
$match_part = parse_url($match);
$match_root =
$match_part["scheme"]."://".$match_part["host"];
$search = array( "|^http://".preg_quote($host)."|i",
"|^(/)|i",
"|^(?!http://)(?!mailto:)|i",
"|/./|",
"|/[^/]+/../|"
);
$replace = array( "",
$match_root."/",
$match."/",
"/",
"/"
);
$expandedLinks = preg_replace($search,$replace,$links);
return $expandedLinks;
}
?
php curl提交头部信息错误
CURLOPT_HTTPHEADER：
一个用来设置HTTP头字段的数组。使用如下的形式的数组进行设置： array('Content-type: text/plain', 'Content-length: 100')
?php
function getwebcontent($url){
$ch = curl_init();
$data = array (
'ap' = '2',
'c1' = '4',
'c2' = '4',
'g_w' = '0100',
'dd' = '0',
'h' = '8',
'iasign' = 'bedvkt2gyd9vkgrx',
'pp' = '200',
);
$headers['X-rvt'] = 'IA401004bedvkt2gyd9vkgrx82lIsT';
$headers['Referer'] = ';c2=4g_w=0100h=1';
$headers['Accept-Language'] = 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3';
$headers['Cookie'] = 'iasign=bedvkt2gyd9vkgrx;';
$timeout = 20;
curl_setopt ($ch, CURLOPT_URL, $url);
curl_setopt ($ch, CURLOPT_POST, 1);
curl_setopt ($ch, CURLOPT_VERBOSE , 0);
curl_setopt ($ch, CURLOPT_HEADER, 1);
curl_setopt ($ch, CURLOPT_RETURNTRANSFER, 0);
curl_setopt ($ch, CURLOPT_CONNECTTIMEOUT, $timeout);
curl_setopt ($ch, CURLOPT_FOLLOWLOCATION, 0);
curl_setopt ($ch, CURLOPT_POSTFIELDS, $data );
curl_setopt ($ch, CURLOPT_HTTPHEADER , $headers);
$contents = curl_exec($ch);
curl_close($ch);
return $contents;
}
$c = getwebcontent('');
print($c);
请问高手，如何解决php的curl内存不够的问题呢？
curl下载的文件内容是可以直接输出到文件，而不是内存，请设置这个选项：
$fp = fopen('temp.jpg', 'w');
curl_setopt($c, CURLOPT_RETURNTRANSFER, false);
curl_setopt($c, CURLOPT_FILE, $fp);
也可以通过设置memory_limit来提高最大内存使用量
ini_set('memory_limit', '1024M');
您的采纳就是我的动力！
