本文目录一览：
python 如何抓取动态页面内容？
输入url，得到html，我早就写了函数了
自己搜：
getUrlRespHtml
就可以找到对应的python函数：
#------------------------------------------------------------------------------
def getUrlResponse(url, postDict={}, headerDict={}, timeout=0, useGzip=False, postDataDelimiter="") :
"""Get response from url, support optional postDict,headerDict,timeout,useGzip
Note:
1. if postDict not null, url request auto become to POST instead of default GET
2 if you want to auto handle cookies, should call initAutoHandleCookies() before use this function.
then following urllib2.Request will auto handle cookies
"""
# makesure url is string, not unicode, otherwise urllib2.urlopen will error
url = str(url);
if (postDict) :
if(postDataDelimiter==""):
postData = urllib.urlencode(postDict);
else:
postData = "";
for eachKey in postDict.keys() :
postData += str(eachKey) + "=" + str(postDict[eachKey]) + postDataDelimiter;
postData = postData.strip();
logging.info("postData=%s", postData);
req = urllib2.Request(url, postData);
logging.info("req=%s", req);
req.add_header('Content-Type', "application/x-www-form-urlencoded");
else :
req = urllib2.Request(url);
defHeaderDict = {
'User-Agent' : gConst['UserAgent'],
'Cache-Control' : 'no-cache',
'Accept' : '*/*',
'Connection' : 'Keep-Alive',
};
# add default headers firstly
for eachDefHd in defHeaderDict.keys() :
#print "add default header: %s=%s"%(eachDefHd,defHeaderDict[eachDefHd]);
req.add_header(eachDefHd, defHeaderDict[eachDefHd]);
if(useGzip) :
#print "use gzip for",url;
req.add_header('Accept-Encoding', 'gzip, deflate');
# add customized header later - allow overwrite default header
if(headerDict) :
#print "added header:",headerDict;
for key in headerDict.keys() :
req.add_header(key, headerDict[key]);
if(timeout 0) :
# set timeout value if necessary
resp = urllib2.urlopen(req, timeout=timeout);
else :
resp = urllib2.urlopen(req);
#update cookies into local file
if(gVal['cookieUseFile']):
gVal['cj'].save();
logging.info("gVal['cj']=%s", gVal['cj']);
return resp;
#------------------------------------------------------------------------------
# get response html==body from url
#def getUrlRespHtml(url, postDict={}, headerDict={}, timeout=0, useGzip=False) :
def getUrlRespHtml(url, postDict={}, headerDict={}, timeout=0, useGzip=True, postDataDelimiter="") :
resp = getUrlResponse(url, postDict, headerDict, timeout, useGzip, postDataDelimiter);
respHtml = resp.read();
#here, maybe, even if not send Accept-Encoding: gzip, deflate
#but still response gzip or deflate, so directly do undecompress
#if(useGzip) :
#print "---before unzip, len(respHtml)=",len(respHtml);
respInfo = resp.info();
# Server: nginx/1.0.8
# Date: Sun, 08 Apr 2012 12:30:35 GMT
# Content-Type: text/html
# Transfer-Encoding: chunked
# Connection: close
# Vary: Accept-Encoding
# ...
# Content-Encoding: gzip
# sometime, the request use gzip,deflate, but actually returned is un-gzip html
# - response info not include above "Content-Encoding: gzip"
# eg:
# - so here only decode when it is indeed is gziped data
#Content-Encoding: deflate
if("Content-Encoding" in respInfo):
if("gzip" == respInfo['Content-Encoding']):
respHtml = zlib.decompress(respHtml, 16+zlib.MAX_WBITS);
elif("deflate" == respInfo['Content-Encoding']):
respHtml = zlib.decompress(respHtml, -zlib.MAX_WBITS);
return respHtml;
及示例代码：
url = "";
respHtml = getUrlRespHtml(url);
完全库函数，自己搜：
crifanLib.py
关于抓取动态页面，详见：
Python专题教程：抓取网站，模拟登陆，抓取动态网页
（自己搜标题即可找到）
python怎么获取动态网页链接？
四中方法：
'''
得到当前页面所有连接
'''
import requests
import re
from bs4 import BeautifulSoup
from lxml import etree
from selenium import webdriver
url = ''
r = requests.get(url)
r.encoding = 'gb2312'
# 利用 re
matchs = re.findall(r"(?=href=\").+?(?=\")|(?=href=\').+?(?=\')" , r.text)
for link in matchs:
print(link)
print()
# 利用 BeautifulSoup4 （DOM树）
soup = BeautifulSoup(r.text,'lxml')
for a in soup.find_all('a'):
link = a['href']
print(link)
print()
# 利用 lxml.etree （XPath）
tree = etree.HTML(r.text)
for link in tree.xpath("//@href"):
print(link)
print()
# 利用selenium（要开浏览器！）
driver = webdriver.Firefox()
driver.get(url)
for link in driver.find_elements_by_tag_name("a"):
print(link.get_attribute("href"))
driver.close()
如何用Python爬取动态加载的网页数据
动态网页抓取都是典型的办法
直接查看动态网页的加载规则。如果是ajax，则将ajax请求找出来给python。 如果是js去处后生成的URL。就要阅读JS，搞清楚规则。再让python生成URL。这就是常用办法
办法2，使用python调用webkit内核的，IE内核，或者是firefox内核的浏览器。然后将浏览结果保存下来。通常可以使用浏览器测试框架。它们内置了这些功能
办法3，通过http proxy，抓取内容并进行组装。甚至可以嵌入自己的js脚本进行hook. 这个方法通常用于系统的反向工程软件
