{
                'skuInfo': [
                    '機(jī)型:iphone X',
                    '顏色:透明白'
                ],
                'frontUserName': 'c****x',
                'frontUserAvatar': 'https://yanxuan.nosdn./c230f2c2a6e7223810755518ce5cd62f',
                'content': '手機(jī)套很薄，裸機(jī)的手感，硅膠的摸著好舒服，看著屏幕都變大了。顏值很高。',
                'createTime': 1554982110981,
                'picList': [
                    'https://yanxuan.nosdn./749cfcb4877be8e7fce449900d763731.jpg',
                    'https://yanxuan.nosdn./6744d04efcd7509556b710bc0d9fa6b0.jpg'
                ],
                'commentReplyVO': null,
                'memberLevel': 4,
                'appendCommentVO': null,
                'star': 5,
                'itemId': 3444035
            }

conn = MongoClient('mongodb://%s:%s@ds149974.mlab.com:49974/you163' % ('you163', 'you163'))
db = conn.you163
mongo_collection = db.iPhone

job_details_uri = job.find('h3', attrs={'class': 'name'}).find('a')['href']
job_company = job.find('div', attrs={'class': 'company-text'}).find('h3', attrs={'class': 'name'}).find('a').text
job_salary = job.find('h3', attrs={'class': 'name'}).find('span', attrs={'class': 'red'}).text

for job in jobs:
    job_dict = {}
    job_details_uri = job.find('h3', attrs={'class': 'name'}).find('a')['href']
    job_company = job.find('div', attrs={'class': 'company-text'}).find('h3', attrs={'class': 'name'}).find('a').text
    job_salary = job.find('h3', attrs={'class': 'name'}).find('span', attrs={'class': 'red'}).text
    job_details = str(job.find('p'))
    print(job_details)
    job_rege = re.match(rege, job_details)
    job_dict['name'] = job_company
    job_dict['uri'] = job_details_uri
    job_dict['salary'] = job_salary
    job_dict['site'] = job_rege.group(1)
    job_dict['year'] = job_rege.group(2)
    job_dict['edu'] = job_rege.group(3)
    job_list.append(job_dict)
print(job_list)

job_conn = MongoClient('mongodb://%s:%s@ds151612.mlab.com:51612/boss' % ('boss', 'boss123'))
job_db = job_conn.boss
job_collection = job_db.boss

details_collection = job_db.job_details


def run_main():
    jobs = job_collection.find()
    for job in jobs:
        print('獲得工作的uri ', job['uri'])
        get_details(job)
        time.sleep(1)


def get_details(items):
    base_url = config.url
    url = base_url + items['uri']
    company_name = items['name']
    try:
        res = requests.get(url, headers=header).text
        content = BeautifulSoup(res, 'html.parser')
        text = content.find('div', attrs={'class': 'text'}).text.strip()
        result = {'name': company_name, 'details': text}
        details_collection.insert_one(result)
    except:
        raise 


if __name__ == '__main__':
    run_main()

s.weibo.com/user?q=林志玲

def get_userinfo(uid):
    try:
        url = 'https://m.weibo.cn/api/container/getIndex?type=uid&value=%s' % uid
        res = requests.get(url).json()
        containerid = res['data']['tabsInfo']['tabs'][1]['containerid']
        mblog_counts = res['data']['userInfo']['statuses_count']
        followers_count = res['data']['userInfo']['followers_count']
        userinfo = {
            'containerid': containerid,
            'mblog_counts': mblog_counts,
            'followers_count': followers_count
        }
        return userinfo
    except:
        raise

def get_blog_by_text(containerid, blog_text, name):
    blog_list = []
    page = 1
    while True:
        try:
            url = 'https://m.weibo.cn/api/container/getIndex?containerid=%s&page=%s' % (containerid, page)
            res_code = requests.get(url).status_code
            if res_code == 418:
                print('訪問太頻繁，過會再試試吧')
                return False
            res = requests.get(url).json()
            cards = res['data']['cards']
            if len(cards) > 0:
                for i in range(len(cards)):
                    print('-----正在爬取第' + str(page) + '頁，第' + str(i+1) + '條微博------')
                    blog_dict = get_blog_info(cards, i, name, page)
                    blog_list.append(blog_dict)
                    if blog_list is False:
                        break
                    mblog_text = blog_dict['mblog_text']
                    create_time = blog_dict['create_time']
                    if blog_text in mblog_text:
                        print('找到相關(guān)微博')
                        return blog_dict['mblog_id']
                    elif checkTime(create_time, config.day) is False:
                        print('沒有找到相關(guān)微博')
                        return blog_list
                page += 1
                time.sleep(config.sleep_time)
            else:
                print('沒有任何微博哦')
                break

        except:
            pass

day = 90  # 最久抓取的微博時(shí)間，60即為只抓取兩個(gè)月前到現(xiàn)在的微博
sleep_time = 5  # 延遲時(shí)間，建議配置5-10s

from weibo_spider import WeiBo
from config import headers


def main(name, spider_type, text, page, iscomment, comment_page):
    print('開始...')
    weibo = WeiBo(name, headers)
    ...


if __name__ == '__main__':
    target_name = input('type the name: ')
    spider_type = input('type spider type(Text or Page): ')
    text = '你好'
    page_count = 10
    iscomment = 'No'
    comment_page_count = 100
    while spider_type not in ('Text', 'text', 'Page', 'page'):
        spider_type = input('type spider type(Text or Page): ')
    ...

import datetime
from config import day


def checkTime(inputtime, day):
    ...


def get_blog_info(cards, i, name, page):
    ...

def download_page(nvshen_id_list):
    for i in nvshen_id_list:
        print('正在下載ID為' + i + '的HTML網(wǎng)頁')
        url = 'https://www.dongqiudi.com/news/%s' % i
        download = DownloadPage()
        html = download.getHtml(url)
        download.saveHtml(i, html)
        time.sleep(2)

class DownloadPage(object):
    def getHtml(self, url):
        html = requests.get(url=url, cookies=config.session, headers=config.header).content
        return html

    def saveHtml(self, file_name, file_content):
        with open('html_page/' + file_name + '.html', 'wb') as f:
            f.write(file_content)

content.find_all('span', attrs={'style': 'color:#ff0000'})

    w_list = ['吳宣儀', '30萬', '826965', '68', '825847',
              'https://img1.dongqiudi.com/fastdfs3/M00/74/54/180x135/crop/-/ChOxM1vIPpOAZT8AAAHza_WMyRk175.png']
    g_list = ['關(guān)之琳', '20萬', '813611', '88', '812559',
              'https://img1.dongqiudi.com/fastdfs3/M00/6B/94/180x135/crop/-/ChOxM1u1gx2AZ7qmAABi3gRdHS8715.jpg']
    t_list = ['佟麗婭', '22萬', '797779', '93', '795697',
              'https://img1.dongqiudi.com/fastdfs3/M00/60/A7/180x135/crop/-/ChOxM1ufUh2AJdR0AABXtcU22fg956.jpg']
    y_list = ['楊丞琳', '7萬', '1173681', '45', '1168209',
              'https://img1./fastdfs4/M00/CA/F7/ChMf8F0pTOKAaefqAA5nOMM0LK0171.jpg']

def get_comment(comment_l):
    client = pymongo.MongoClient('mongodb://douban:douban1989@ds149744.mlab.com:49744/douban')
    db = client.douban
    mongo_collection = db.comment
    comment_list = []
    comment = []
    print('Save to MongoDB')
    for i in comment_l:
        response = requests.get(i).text
        content = BeautifulSoup(response, 'html.parser')
        tmp_list = content.find_all('div', attrs={'class': 'comment-item'})
        comment_list = comment_list + tmp_list
        for k in comment_list:
            tmp_comment = k.find('p').text
            mongo_collection.insert_one({'comment': tmp_comment})
            comment.append(tmp_comment)
    print('Save Finish!')