网页数据抓取怎么写(之前用C#帮朋友写了一个抓取网页信息的程序)

　　网页数据抓取怎么写(之前用C#帮朋友写了一个抓取网页信息的程序)

　　我用C#帮朋友写了一个抓取网页信息的程序。这是非常复杂的。今天朋友要下载网页资料。有很多，想偷懒，又不想用C#了，就想到了Python。两个小时，在记事本中输入，然后在 IDLE（Python GUI）中测试。发现Python等解释型语言非常好，不用编译，直接写个脚本就行了。代码显示如下：

　　# -*- coding:gb2312 -*-

import sys

import urllib

import re

#从html中解析标题

def ParshTitle(html):

startPos = html.find(\'\')

endpos = html.find(\'\')

strTmp = html[startPos+29:endpos]

strTmp = strTmp.replace(\'\', \'\')

return strTmp

#从html中解析CPI数据

def ParshCPI(html):

startPos = html.find(\'\')

endpos = html.find(\'\')

strTmp = html[startPos:endpos]

return "" + strTmp + ""

#从html中解析城镇投资数据

def ParshUrbanInvestment(html):

startPos = html.find(\'\')

endpos = html.find(\'\')

strTmp = html[startPos:endpos]

return "" + strTmp + ""

#提取各地区数据

def GetHtmlData(url, htmlfile, bisCPI):

wp = urllib.urlopen(url)#打开连接

content = wp.read() #获取页面内容

content = content.replace(\'\r\n\', \'\')

title = ParshTitle(content)

if bisCPI:

content = ParshCPI(content)

else:

content = ParshUrbanInvestment(content)

fl = title + htmlfile

#将文件路径转为gbk编码

fl = unicode(fl,\'gbk\')

f = open(fl, \'w\')

f.write(content)

f.close()

if __name__ =="__main__":

#首先提取CPI数据

num_list = range(72) #生成0～71的数字

strUrl = "http://www.stats.gov.cn/was40/gjtjj_detail_data.jsp?searchword=%28docTitle%21%3D%B9%FA%*敏*感*词*%DA%C9%FA%B2%FA%D7%DC%D6%B5+and+docTitle%21%3D%B3%C7%D5%F2%B5%A5%CE%BB%B4%D3%D2%B5%C8%CB%D4%B1%C0%CD%B6%AF%B1%A8%B3%EA+and+docTitle%21%3D%C5%A9%C1%D6%*敏*感*词*%C1%D3%E6%D2%B5%D7%DC%B2%FA%D6%B5+and+docTitle%21%3D%B8%F7%B5%D8%C7%F8%C5%A9%C1%D6%*敏*感*词*%C1%D3%E6%D2%B5%D7%DC%B2%FA%D6%B5+and+docTitle%21%3D%D2%DA%D4%AA%D2%D4%C9%CF%C9%CC%C6%B7%BD%BB%D2%D7%CA%D0%B3%A1%D6%F7%D2%AA%D6%B8%B1%EA+and+docTitle%21%3D%B8%F7%B5%D8%C7%F8%D2%DA%D4%AA%D2%D4%C9%CF%C9%CC%C6%B7%BD%BB%D2%D7%CA%D0%B3%A1%BB%F9%B1%BE%C7%E9%BF%F6+and+docTitle%21%3D%B8%F7%B5%D8%C7%F8%C5%A9%B4%E5%BE%D3%C3%F1%BC%D2%CD%A5%C6%BD%BE%F9%C3%BF%C8%CB%CF%D6%BD%F0%CA%D5%C8%EB+and+docTitle%21%3D%B7%D6%B5%D8%C7%F8%B3%C7%D5%F2%B5%A5%CE%BB%BE%CD%D2%B5%C8%CB%D4%B1+and+docTitle%21%3D%B7%D6%B5%D8%C7%F8%B3%C7%D5%F2%B5%A5%CE%BB%BE%CD%D2%B5%C8%CB%D4%B1%C0%CD%B6%AF%B1%A8%B3%EA+and+docTitle%21%3D%C8%AB%B9%FA%D6%F7%D2%AA%C5%A9%B2%FA%C6%B7%C9%FA%B2%FA%BC%DB%B8%F1%D6%B8%CA%FD+and+docTitle%21%3D%B8%F7%B5%D8%C7%F8%C5%A9%B4%E5%BE%D3%C3%F1%BC%D2%CD%A5%C6%BD%BE%F9%C3%BF%C8%CB%CF%D6%BD%F0%D6%A7%B3%F6+and+docTitle%21%3D%C6%F3%D2%B5%BE%B0%C6%F8%D6%B8%CA%FD+and+docTitle%21%3D%*敏*感*词*%BF%C2%BC+and++docTitle%21%3D%B7%D6%B5%D8%C7%F8%B3%C7%D5%F2%B5%A5%CE%BB%BE%CD%D2%B5%C8%CB%D4%B1+and+docTitle%21%3D%B7%D6%B5%D8%C7%F8%B3%C7%D5%F2%B5%A5%CE%BB%BE%CD%D2%B5%C8%CB%D4%B1%C0%CD%B6%AF%B1%A8%B3%EA+and++docTitle%21%3D%C8%AB%B9%FA%D6%F7%D2%AA%C5%A9%B2%FA%C6%B7%C9%FA%B2%FA%BC%DB%B8%F1%D6%B8%CA%FD+and+docTitle%21%3D%B8%F7%B5%D8%C7%F8%C5%A9%B4%E5%BE%D3%C3%F1%BC%D2%CD%A5%C6%BD%BE%F9%C3%BF%C8%CB%CF%D6%BD%F0%D6%A7%B3%F6+and++docTitle%21%3D%B8%F7%B5%D8%C7%F8%B3%C7%D5%F2%BE%D3%C3%F1%BC%D2%CD%A5%CA%D5%D6%A7%BB%F9%B1%BE%C7%E9%BF%F6%29+and+%28docTitle%3D%B8%F7%B5%D8%C7%F8%BE%D3%C3%F1%CF%FB%B7%D1%BC%DB%B8%F1%D6%B8%CA%FD+or+DOCHTMLCONTENT%3D%B8%F7%B5%D8%C7%F8%BE%D3%C3%F1%CF%FB%B7%D1%BC%DB%B8%F1%D6%B8%CA%FD%29&channelid=9951&record="

for i in num_list:

if i==0:

continue

strTemp = strUrl + str(i)

strTxt = "_CPI.htm"

GetHtmlData(strTemp, strTxt, True)

print (str(i) + "/72")

#再提取城镇投资数据

num_list = range(56) #生成0～55的数字

strUrl = "http://www.stats.gov.cn/was40/gjtjj_detail_data.jsp?searchword=%B8%F7%B5%D8%C7%F8%B3%C7%D5%F2%CD%B6%D7%CA&presearchword=%28docTitle%21%3D%B9%FA%*敏*感*词*%DA%C9%FA%B2%FA%D7%DC%D6%B5+and+docTitle%21%3D%B3%C7%D5%F2%B5%A5%CE%BB%B4%D3%D2%B5%C8%CB%D4%B1%C0%CD%B6%AF%B1%A8%B3%EA+and+docTitle%21%3D%C5%A9%C1%D6%*敏*感*词*%C1%D3%E6%D2%B5%D7%DC%B2%FA%D6%B5+and+docTitle%21%3D%B8%F7%B5%D8%C7%F8%C5%A9%C1%D6%*敏*感*词*%C1%D3%E6%D2%B5%D7%DC%B2%FA%D6%B5+and+docTitle%21%3D%D2%DA%D4%AA%D2%D4%C9%CF%C9%CC%C6%B7%BD%BB%D2%D7%CA%D0%B3%A1%D6%F7%D2%AA%D6%B8%B1%EA+and+docTitle%21%3D%B8%F7%B5%D8%C7%F8%D2%DA%D4%AA%D2%D4%C9%CF%C9%CC%C6%B7%BD%BB%D2%D7%CA%D0%B3%A1%BB%F9%B1%BE%C7%E9%BF%F6+and+docTitle%21%3D%B8%F7%B5%D8%C7%F8%C5%A9%B4%E5%BE%D3%C3%F1%BC%D2%CD%A5%C6%BD%BE%F9%C3%BF%C8%CB%CF%D6%BD%F0%CA%D5%C8%EB+and+docTitle%21%3D%B7%D6%B5%D8%C7%F8%B3%C7%D5%F2%B5%A5%CE%BB%BE%CD%D2%B5%C8%CB%D4%B1+and+docTitle%21%3D%B7%D6%B5%D8%C7%F8%B3%C7%D5%F2%B5%A5%CE%BB%BE%CD%D2%B5%C8%CB%D4%B1%C0%CD%B6%AF%B1%A8%B3%EA+and+docTitle%21%3D%C8%AB%B9%FA%D6%F7%D2%AA%C5%A9%B2%FA%C6%B7%C9%FA%B2%FA%BC%DB%B8%F1%D6%B8%CA%FD+and+docTitle%21%3D%B8%F7%B5%D8%C7%F8%C5%A9%B4%E5%BE%D3%C3%F1%BC%D2%CD%A5%C6%BD%BE%F9%C3%BF%C8%CB%CF%D6%BD%F0%D6%A7%B3%F6+and+docTitle%21%3D%C6%F3%D2%B5%BE%B0%C6%F8%D6%B8%CA%FD+and+docTitle%21%3D%*敏*感*词*%BF%C2%BC+and++docTitle%21%3D%B7%D6%B5%D8%C7%F8%B3%C7%D5%F2%B5%A5%CE%BB%BE%CD%D2%B5%C8%CB%D4%B1+and+docTitle%21%3D%B7%D6%B5%D8%C7%F8%B3%C7%D5%F2%B5%A5%CE%BB%BE%CD%D2%B5%C8%CB%D4%B1%C0%CD%B6%AF%B1%A8%B3%EA+and++docTitle%21%3D%C8%AB%B9%FA%D6%F7%D2%AA%C5%A9%B2%FA%C6%B7%C9%FA%B2%FA%BC%DB%B8%F1%D6%B8%CA%FD+and+docTitle%21%3D%B8%F7%B5%D8%C7%F8%C5%A9%B4%E5%BE%D3%C3%F1%BC%D2%CD%A5%C6%BD%BE%F9%C3%BF%C8%CB%CF%D6%BD%F0%D6%A7%B3%F6+and++docTitle%21%3D%B8%F7%B5%D8%C7%F8%B3%C7%D5%F2%BE%D3%C3%F1%BC%D2%CD%A5%CA%D5%D6%A7%BB%F9%B1%BE%C7%E9%BF%F6%29+and+%28docTitle%3D%B8%F7%B5%D8%C7%F8%BE%D3%C3%F1%CF%FB%B7%D1%BC%DB%B8%F1%D6%B8%CA%FD+or+DOCHTMLCONTENT%3D%B8%F7%B5%D8%C7%F8%BE%D3%C3%F1%CF%FB%B7%D1%BC%DB%B8%F1%D6%B8%CA%FD%29&channelid=9951&record="

for i in num_list:

if i==0:

continue

strTemp = strUrl + str(i)

strTxt = "_UI.htm"

GetHtmlData(strTemp, strTxt, False)

print (str(i) + "/56")

　　对上面代码稍微解释一下：这个主要是用来抓取国家统计局网站的统计数据，包括各地区的居民消费价格指数和各地区的城市投资数据。同学们要的数据是2006年1月到现在，在统计局网站上查询发现所有的数据URL都是一样的，只是最后一个id值从1递增，最近是1.，之前时间慢慢增加。统计局的这个网址好长啊！！

　　代码中写了三个函数，分别是获取网页的Tilte，作为保存文件的文件名。

　　一个功能是从html中分析出各个地区的居民消费价格指数，另一个是提取各个地区的城市投资数据。实际上，当前代码的两个功能是相同的。我比较懒，没有提取里面的详细信息。Table的tbody取出来了，哈哈。如果可以，可以使用正则表达式只提取里面的数据。

　　上次用C#写了好久。Python真的很不一样。如果写一个简单的工具，更推荐Python。一是简单，二是不需要编译，直接写一个txt放到解释器里就可以了。NS。