Python采集网页数据保存到excel

今天开始整理写过的Python代码,同时共享出来,需要对大家有帮助。

今天共享的是通过Python采集网页上的数据,然后保存到Excel。思路很简单,用urllib读取网页,然后用Py-excel写excel。

import urllib
from xlwt import Workbook
import datetime

def FetchData():
    book = Workbook(encoding='gbk')    #如果采集数据有中文,需要添加这个 
    sheet1 = book.add_sheet('Sheet 2') #表格缓存 

    i = 0
    theday = datetime.date(2009,12,31)
    while i < 100: #这边的场景就是采集100个网页,每个网址都包含日期 
        i += 1
        theday = theday + datetime.timedelta(days = 1)
        print theday
        theday_str = str(theday)
        sheet1.write(i,0,theday_str)  #写表格 
        check_url = r'http://www.xxx.com/index?date=' + theday_str #网页地址
        try:
            checkfile = urllib.urlopen(check_url)  #网页保存为文本文件 
        except Exception,e:
            print e
            return

        type = sys.getfilesystemencoding()
        for line in checkfile:
            line = line.decode("UTF-8").encode(type)     #网页编码为UTF-8 
            date_west = getdata('date_west', line)       #获取特定数据 
            if date_west != False:
                sheet1.write(i,1,date_west)

    book.save('simple.xls')  #保存excel文件 
    print 'finish!'

'if keywords in the line, get data from > to </'
def getdata(keywords, line):
    data = ''
    if keywords in line:
        start = line.find('>',)
        end = line.find('</', start)
        data = line[start+1:end]
        return data
    return False

Submit a comment

Allowed HTML tags: <a href="http://google.com">google</a> <strong>bold</strong> <em>emphasized</em> <code>code</code> <blockquote>
quote
</blockquote>
example: http://google.com