LaGou/LaGou.py

#!usr/bin/env python
# -*-coding:utf-8 -*-
__author__='WYY'
__date__='2017.03.29'

#实战小项目：爬取拉钩网并作小型数据分析
import requests
import json
import xlwt
import time
import random

class Spider():
    def __init__(self):
        self.keyword=raw_input(u'请输入职位：')

    #获取数据
    def getData(self,url):
        user_agents=['Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20130406 Firefox/23.0',
                       'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0',
                       'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533+ \(KHTML, like Gecko) Element Browser 5.0',
                       'IBM WebExplorer /v0.94', 'Galaxy/1.0 [en] (Mac OS X 10.5.6; U; en)',
                       'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)',
                       'Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14',
                       'Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) \Version/6.0 Mobile/10A5355d Safari/8536.25',
                       'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) \Chrome/28.0.1468.0 Safari/537.36',
                       'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; TheWorld)']
        index=random.randint(0, 9)
        user_agent=user_agents[index]
        #cookie要自己抓
        headers={'User_agent':user_agent,
                 'cookie':'your cookies'}
        html=requests.get(url,headers=headers)
        data=json.loads(html.text)
        return data

    #获取职位信息并存入列表
    def getPosition(self,url):
        data=self.getData(url)
        position=data['content']['positionResult']['result']
        po_list=[]
        if position is not None:
            for i in position:
                main=[]
                main.append(i['companyFullName'])
                main.append(i['financeStage'])
                main.append(i['positionName'])
                main.append(i['positionLables'])
                main.append(i['salary'])
                main.append(i['city'])
                main.append(i['education'])
                main.append(i['workYear'])
                main.append(i['jobNature'])
                main.append(i['createTime'])
                po_list.append(main)
        return po_list

    #获取数据,存入一个大的list
    def saveDetail(self):
        self.New=int(raw_input(u'请输入要爬取的页数:'))
        self.time=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        print u'\n本地时间：',self.time
        print u'\n开始采集数据...'
        container=[]
        for page in range(1,self.New+1):
            self.url='https://www.lagou.com/jobs/positionAjax.json?px=new&first=true&pn='+str(page)+'&kd='+str(self.keyword)
            po_list=self.getPosition(self.url)
            time.sleep(3)
            print u'第',page,u'项完毕'
            container=container+po_list
        return container

    #将数据存入excel
    def saveAll(self):
        book=xlwt.Workbook()
        sheet =book.add_sheet(str(self.keyword), cell_overwrite_ok=True)
        container=self.saveDetail()
        print u'\n采集完毕'
        print u'\n准备将数据存入表格...'
        heads=[u'公司全名', u'融资状况', u'工作名称', u'标签', u'薪酬', u'城市', u'学历要求',u'经验要求',u'工作类型',u'数据创建时间']
        ii=0
        for head in heads:
            sheet.write(0,ii,head)
            ii+=1
        i=1
        for list in container:
            j=0
            for one in list:
                sheet.write(i, j, one)
                j+=1
            i+=1
        book.save(str(self.keyword)+'.xls')
        print u'\n录入成功！'

spider=Spider()
spider.saveAll()