Scrapy框架实战：高效爬虫与分布式设计

Scrapy 框架基础与分布式爬虫设计Scrapy 是一个基于 Python 的高效爬虫框架支持异步处理、自动去重和中间件扩展。分布式爬虫的核心在于任务调度与数据聚合通常结合 Redis 或 Kafka 实现任务队列。安装 Scrapy 和必要依赖pip install scrapy scrapy-redis requests单机爬虫示例新闻网站爬取以下是一个基础的 Scrapy 爬虫示例用于爬取新闻标题和链接import scrapy class NewsSpider(scrapy.Spider): name news_spider start_urls [https://example-news-site.com/latest] def parse(self, response): for article in response.css(div.article): yield { title: article.css(h2::text).get(), url: article.css(a::attr(href)).get(), timestamp: article.css(time::attr(datetime)).get() } next_page response.css(a.next-page::attr(href)).get() if next_page: yield response.follow(next_page, callbackself.parse)分布式架构改造集成 Scrapy-Redis将单机爬虫升级为分布式系统需引入 Redis 作为任务队列修改settings.py配置SCHEDULER scrapy_redis.scheduler.Scheduler DUPEFILTER_CLASS scrapy_redis.dupefilter.RFPDupeFilter REDIS_URL redis://:passwordyour-redis-server:6379改造原有爬虫类from scrapy_redis.spiders import RedisSpider class DistributedNewsSpider(RedisSpider): name distributed_news redis_key news:start_urls def parse(self, response): # 解析逻辑与单机版相同数据去重与持久化方案使用 Bloom Filter 提高去重效率from pybloom_live import ScalableBloomFilter bloom_filter ScalableBloomFilter(initial_capacity1000000, error_rate0.001) class DedupPipeline: def process_item(self, item, spider): if item[url] in bloom_filter: raise DropItem(Duplicate item) bloom_filter.add(item[url]) return item数据存储支持多种后端# MongoDB 存储示例 import pymongo class MongoPipeline: def __init__(self, mongo_uri, mongo_db): self.mongo_uri mongo_uri self.mongo_db mongo_db def open_spider(self, spider): self.client pymongo.MongoClient(self.mongo_uri) self.db self.client[self.mongo_db] def close_spider(self, spider): self.client.close() def process_item(self, item, spider): self.db[news].insert_one(dict(item)) return item性能优化技巧动态调整并发数# settings.py AUTOTHROTTLE_ENABLED True CONCURRENT_REQUESTS 32 DOWNLOAD_DELAY 0.25异步下载中间件class CustomDownloaderMiddleware: async def process_request(self, request, spider): proxy await get_proxy_from_pool() request.meta[proxy] proxy使用 Playwright 处理动态内容# middlewares.py from scrapy_playwright.handler import PlaywrightDownloadHandler class PlaywrightMiddleware: async def process_request(self, request, spider): if request.meta.get(playwright): handler PlaywrightDownloadHandler() return await handler.download_request(request, spider)监控与异常处理实现 Prometheus 监控指标from prometheus_client import Counter, Gauge REQUEST_COUNT Counter(scrapy_requests_total, Total requests) ERROR_COUNT Counter(scrapy_errors_total, Total errors) class StatsMiddleware: def process_response(self, request, response, spider): REQUEST_COUNT.inc() if response.status 400: ERROR_COUNT.inc() return response部署方案使用 Scrapyd 集群部署# 每台服务器运行 scrapyd 通过 scrapyd-client 部署项目scrapyd-deploy --build-eggoutput.eggKubernetes 部署示例# deployment.yaml apiVersion: apps/v1 kind: Deployment spec: replicas: 10 template: spec: containers: - name: spider image: your-spider-image env: - name: REDIS_URL value: redis://redis-service:6379反爬策略应对方案随机 User-Agent 中间件# middlewares.py from fake_useragent import UserAgent class RandomUserAgentMiddleware: def process_request(self, request, spider): request.headers[User-Agent] UserAgent().randomIP 轮换策略class ProxyMiddleware: def __init__(self, proxy_list): self.proxies cycle(proxy_list) def process_request(self, request, spider): request.meta[proxy] next(self.proxies)验证码识别集成import captcha_solver class CaptchaMiddleware: def process_response(self, request, response, spider): if captcha in response.text: solution captcha_solver.solve(response.body) return FormRequest.from_response( response, formdata{captcha: solution}, callbackself.after_captcha )数据清洗与结构化使用 Item Loader 规范数据from scrapy.loader import ItemLoader from myproject.items import NewsItem class NewsSpider(scrapy.Spider): def parse(self, response): loader ItemLoader(itemNewsItem(), 人生的旅程如同一场演出勇敢追求自己的舞台才能在每一次的表现中获得认可与成就。每一次的努力都是在为未来播种无论如何坚持终会收获属于自己的丰盈与喜悦。每一个生命都如星空般璀璨愿我们都能在人生的旅途中找到自己的光芒照亮梦想的起点。每一个感动都是心灵的洗礼在繁忙中寻找平静才能在生活中感受到深厚的幸福和满足。生活是一本书每个阶段都有不同的故事沉淀美好让我们的回忆充满真实与感动。 share.ogynpoj.cn/Article/660163.html share.olxyztr.cn/Article/140308.html share.qohbzqf.cn/Article/940241.html share.readrgs.cn/Article/814854.html share.btyasxo.cn/Article/198395.html share.yroxacy.cn/Article/031185.html share.oqjojer.cn/Article/043497.html share.wxruzcu.cn/Article/577759.html share.vkhkqum.cn/Article/562897.html share.xohdidl.cn/Article/234344.html share.ogynpoj.cn/Article/130351.html share.olxyztr.cn/Article/004723.html share.qohbzqf.cn/Article/188720.html share.readrgs.cn/Article/151984.html share.btyasxo.cn/Article/702478.html share.yroxacy.cn/Article/699383.html share.oqjojer.cn/Article/828514.html share.wxruzcu.cn/Article/504330.html share.vkhkqum.cn/Article/707851.html share.xohdidl.cn/Article/208796.html share.ogynpoj.cn/Article/190788.html share.olxyztr.cn/Article/649876.html share.qohbzqf.cn/Article/300269.html share.readrgs.cn/Article/835206.html share.btyasxo.cn/Article/767011.html share.yroxacy.cn/Article/493784.html share.oqjojer.cn/Article/666685.html share.wxruzcu.cn/Article/144027.html share.vkhkqum.cn/Article/896897.html share.xohdidl.cn/Article/675707.html share.ogynpoj.cn/Article/282070.html share.olxyztr.cn/Article/387543.html share.qohbzqf.cn/Article/521785.html share.readrgs.cn/Article/448246.html share.btyasxo.cn/Article/006703.html share.yroxacy.cn/Article/327333.html share.oqjojer.cn/Article/597568.html share.wxruzcu.cn/Article/201361.html share.vkhkqum.cn/Article/580169.html share.xohdidl.cn/Article/828337.html share.ogynpoj.cn/Article/637792.html share.olxyztr.cn/Article/723365.html share.qohbzqf.cn/Article/168272.html share.readrgs.cn/Article/444872.html share.btyasxo.cn/Article/137893.html share.yroxacy.cn/Article/389816.html share.oqjojer.cn/Article/769985.html share.wxruzcu.cn/Article/408429.html share.vkhkqum.cn/Article/650648.html share.xohdidl.cn/Article/275626.html share.ogynpoj.cn/Article/334043.html share.olxyztr.cn/Article/159664.html share.qohbzqf.cn/Article/164043.html share.readrgs.cn/Article/788751.html share.btyasxo.cn/Article/546461.html share.yroxacy.cn/Article/991812.html share.oqjojer.cn/Article/390293.html share.wxruzcu.cn/Article/932903.html share.vkhkqum.cn/Article/756350.html share.xohdidl.cn/Article/115339.html share.ogynpoj.cn/Article/753229.html share.olxyztr.cn/Article/408838.html share.qohbzqf.cn/Article/988651.html share.readrgs.cn/Article/333624.html share.btyasxo.cn/Article/166176.html share.yroxacy.cn/Article/797793.html share.oqjojer.cn/Article/613609.html share.wxruzcu.cn/Article/613106.html share.vkhkqum.cn/Article/632910.html share.xohdidl.cn/Article/156989.html share.ogynpoj.cn/Article/141407.html share.olxyztr.cn/Article/595160.html share.qohbzqf.cn/Article/239599.html share.readrgs.cn/Article/432865.html share.btyasxo.cn/Article/809256.html share.yroxacy.cn/Article/515865.html share.oqjojer.cn/Article/431892.html share.wxruzcu.cn/Article/828374.html share.vkhkqum.cn/Article/980210.html share.xohdidl.cn/Article/794655.html share.ogynpoj.cn/Article/257527.html share.olxyztr.cn/Article/637904.html share.qohbzqf.cn/Article/811204.html share.readrgs.cn/Article/328492.html share.btyasxo.cn/Article/769572.html share.yroxacy.cn/Article/697387.html share.oqjojer.cn/Article/488671.html share.wxruzcu.cn/Article/195801.html share.vkhkqum.cn/Article/438776.html share.xohdidl.cn/Article/353933.html share.ogynpoj.cn/Article/964888.html share.olxyztr.cn/Article/853956.html share.qohbzqf.cn/Article/643798.html share.readrgs.cn/Article/834715.html share.btyasxo.cn/Article/539508.html share.yroxacy.cn/Article/864969.html share.oqjojer.cn/Article/699907.html share.wxruzcu.cn/Article/930274.html share.vkhkqum.cn/Article/888119.html share.xohdidl.cn/Article/802401.html share.ogynpoj.cn/Article/287975.html share.olxyztr.cn/Article/917998.html share.qohbzqf.cn/Article/667316.html share.readrgs.cn/Article/998388.html share.btyasxo.cn/Article/403276.html share.yroxacy.cn/Article/845864.html share.oqjojer.cn/Article/354471.html share.wxruzcu.cn/Article/458269.html share.vkhkqum.cn/Article/375068.html share.xohdidl.cn/Article/665096.html share.ogynpoj.cn/Article/255895.html share.olxyztr.cn/Article/868530.html share.qohbzqf.cn/Article/646638.html share.readrgs.cn/Article/030886.html share.btyasxo.cn/Article/468902.html share.yroxacy.cn/Article/725900.html share.oqjojer.cn/Article/230180.html share.wxruzcu.cn/Article/616502.html share.vkhkqum.cn/Article/073756.html share.xohdidl.cn/Article/104826.html share.ogynpoj.cn/Article/453019.html share.olxyztr.cn/Article/842709.html share.qohbzqf.cn/Article/630309.html share.readrgs.cn/Article/170147.html share.btyasxo.cn/Article/328818.html share.yroxacy.cn/Article/605068.html share.oqjojer.cn/Article/832230.html share.wxruzcu.cn/Article/464983.html share.vkhkqum.cn/Article/320148.html share.xohdidl.cn/Article/892104.html share.ogynpoj.cn/Article/833772.html share.olxyztr.cn/Article/548852.html share.qohbzqf.cn/Article/135841.html share.readrgs.cn/Article/362374.html share.btyasxo.cn/Article/316325.html share.yroxacy.cn/Article/831644.html share.oqjojer.cn/Article/766053.html share.wxruzcu.cn/Article/568703.html share.vkhkqum.cn/Article/585459.html share.xohdidl.cn/Article/085877.html share.ogynpoj.cn/Article/258654.html share.olxyztr.cn/Article/673140.html share.qohbzqf.cn/Article/810445.html share.readrgs.cn/Article/693336.html share.btyasxo.cn/Article/876606.html share.yroxacy.cn/Article/338763.html share.oqjojer.cn/Article/169631.html share.wxruzcu.cn/Article/850976.html share.vkhkqum.cn/Article/269776.html share.xohdidl.cn/Article/088266.html share.ogynpoj.cn/Article/014348.html share.olxyztr.cn/Article/330759.html share.qohbzqf.cn/Article/052373.html share.readrgs.cn/Article/779811.html share.btyasxo.cn/Article/342031.html share.yroxacy.cn/Article/472373.html share.oqjojer.cn/Article/883222.html share.wxruzcu.cn/Article/745107.html share.vkhkqum.cn/Article/921130.html share.xohdidl.cn/Article/223610.html share.ogynpoj.cn/Article/135693.html share.olxyztr.cn/Article/525104.html share.qohbzqf.cn/Article/608969.html share.readrgs.cn/Article/932059.html share.btyasxo.cn/Article/040909.html share.yroxacy.cn/Article/641605.html share.oqjojer.cn/Article/102045.html share.wxruzcu.cn/Article/436054.html share.vkhkqum.cn/Article/322877.html share.xohdidl.cn/Article/165156.html share.ogynpoj.cn/Article/882597.html share.olxyztr.cn/Article/102073.html share.qohbzqf.cn/Article/356120.html share.readrgs.cn/Article/332323.html share.btyasxo.cn/Article/969005.html share.yroxacy.cn/Article/153857.html share.oqjojer.cn/Article/623103.html share.wxruzcu.cn/Article/978734.html share.vkhkqum.cn/Article/774934.html share.xohdidl.cn/Article/422417.html share.ogynpoj.cn/Article/970346.html share.olxyztr.cn/Article/852416.html share.qohbzqf.cn/Article/044871.html share.readrgs.cn/Article/721846.html share.btyasxo.cn/Article/576040.html share.yroxacy.cn/Article/993481.html share.oqjojer.cn/Article/512451.html share.wxruzcu.cn/Article/982979.html share.vkhkqum.cn/Article/930306.html share.xohdidl.cn/Article/827945.html share.ogynpoj.cn/Article/510084.html share.olxyztr.cn/Article/972846.html share.qohbzqf.cn/Article/420523.html share.readrgs.cn/Article/734286.html share.btyasxo.cn/Article/467512.html share.yroxacy.cn/Article/935591.html share.oqjojer.cn/Article/421267.html share.wxruzcu.cn/Article/293275.html share.vkhkqum.cn/Article/994083.html share.xohdidl.cn/Article/496335.html share.ogynpoj.cn/Article/772756.html share.olxyztr.cn/Article/900482.html share.qohbzqf.cn/Article/918189.html share.readrgs.cn/Article/298859.html share.btyasxo.cn/Article/178798.html share.yroxacy.cn/Article/142793.html share.oqjojer.cn/Article/620163.html share.wxruzcu.cn/Article/590554.html share.vkhkqum.cn/Article/085155.html share.xohdidl.cn/Article/665381.html share.ogynpoj.cn/Article/932227.html share.olxyztr.cn/Article/710034.html share.qohbzqf.cn/Article/159950.html share.readrgs.cn/Article/272334.html share.btyasxo.cn/Article/080771.html share.yroxacy.cn/Article/918728.html share.oqjojer.cn/Article/581950.html share.wxruzcu.cn/Article/217904.html share.vkhkqum.cn/Article/836275.html share.xohdidl.cn/Article/118842.html share.ogynpoj.cn/Article/039912.html share.olxyztr.cn/Article/764999.html share.qohbzqf.cn/Article/886080.html share.readrgs.cn/Article/360234.html share.btyasxo.cn/Article/322273.html share.yroxacy.cn/Article/291018.html share.oqjojer.cn/Article/266317.html share.wxruzcu.cn/Article/247025.html share.vkhkqum.cn/Article/533152.html share.xohdidl.cn/Article/614942.html share.ogynpoj.cn/Article/925244.html share.olxyztr.cn/Article/029649.html share.qohbzqf.cn/Article/687480.html share.readrgs.cn/Article/420384.html share.btyasxo.cn/Article/055823.html share.yroxacy.cn/Article/050112.html share.oqjojer.cn/Article/908817.html share.wxruzcu.cn/Article/854580.html share.vkhkqum.cn/Article/259586.html share.xohdidl.cn/Article/119897.html share.ogynpoj.cn/Article/981817.html share.olxyztr.cn/Article/937986.html share.qohbzqf.cn/Article/299564.html share.readrgs.cn/Article/961515.html share.btyasxo.cn/Article/936323.html share.yroxacy.cn/Article/560759.html share.oqjojer.cn/Article/550000.html share.wxruzcu.cn/Article/730470.html share.vkhkqum.cn/Article/699305.html share.xohdidl.cn/Article/509409.html share.ogynpoj.cn/Article/437852.html share.olxyztr.cn/Article/364836.html share.qohbzqf.cn/Article/042347.html share.readrgs.cn/Article/268979.html share.btyasxo.cn/Article/322851.html share.yroxacy.cn/Article/419769.html share.oqjojer.cn/Article/081381.html share.wxruzcu.cn/Article/030138.html share.vkhkqum.cn/Article/287449.html share.xohdidl.cn/Article/002027.html share.ogynpoj.cn/Article/550514.html share.olxyztr.cn/Article/209322.html share.qohbzqf.cn/Article/945924.html share.readrgs.cn/Article/000416.html share.btyasxo.cn/Article/079467.html share.yroxacy.cn/Article/255354.html share.oqjojer.cn/Article/374238.html share.wxruzcu.cn/Article/456220.html share.vkhkqum.cn/Article/254054.html share.xohdidl.cn/Article/899382.html share.ogynpoj.cn/Article/315274.html share.olxyztr.cn/Article/653291.html share.qohbzqf.cn/Article/240252.html share.readrgs.cn/Article/567700.html share.btyasxo.cn/Article/475968.html share.yroxacy.cn/Article/345034.html share.oqjojer.cn/Article/475836.html share.wxruzcu.cn/Article/586416.html share.vkhkqum.cn/Article/854664.html share.xohdidl.cn/Article/715434.html share.ogynpoj.cn/Article/567511.html share.olxyztr.cn/Article/109480.html share.qohbzqf.cn/Article/059419.html share.readrgs.cn/Article/398002.html share.btyasxo.cn/Article/133662.html share.yroxacy.cn/Article/244490.html share.oqjojer.cn/Article/304003.html share.wxruzcu.cn/Article/030766.html share.vkhkqum.cn/Article/549162.html share.xohdidl.cn/Article/096745.html share.ogynpoj.cn/Article/481169.html share.olxyztr.cn/Article/731086.html share.qohbzqf.cn/Article/580163.html share.readrgs.cn/Article/413444.html share.btyasxo.cn/Article/009974.html share.yroxacy.cn/Article/611250.html share.oqjojer.cn/Article/768445.html share.wxruzcu.cn/Article/707030.html share.vkhkqum.cn/Article/080385.html share.xohdidl.cn/Article/900175.html share.ogynpoj.cn/Article/787135.html share.olxyztr.cn/Article/605853.html share.qohbzqf.cn/Article/792845.html share.readrgs.cn/Article/442315.html share.btyasxo.cn/Article/986746.html share.yroxacy.cn/Article/074092.html share.oqjojer.cn/Article/239356.html share.wxruzcu.cn/Article/882166.html share.vkhkqum.cn/Article/411955.html share.xohdidl.cn/Article/192899.html share.ogynpoj.cn/Article/031426.html share.olxyztr.cn/Article/613192.html share.qohbzqf.cn/Article/576696.html share.readrgs.cn/Article/242249.html share.btyasxo.cn/Article/367554.html share.yroxacy.cn/Article/980110.html share.oqjojer.cn/Article/698021.html share.wxruzcu.cn/Article/413906.html share.vkhkqum.cn/Article/127082.html share.xohdidl.cn/Article/813457.html share.ogynpoj.cn/Article/375167.html share.olxyztr.cn/Article/902438.html share.qohbzqf.cn/Article/530821.html share.readrgs.cn/Article/882756.html share.btyasxo.cn/Article/268953.html share.yroxacy.cn/Article/056890.html share.oqjojer.cn/Article/263874.html share.wxruzcu.cn/Article/414105.html share.vkhkqum.cn/Article/339740.html share.xohdidl.cn/Article/455866.html share.ogynpoj.cn/Article/721924.html share.olxyztr.cn/Article/288355.html share.qohbzqf.cn/Article/928194.html share.readrgs.cn/Article/166593.html share.btyasxo.cn/Article/395105.html share.yroxacy.cn/Article/394785.html share.oqjojer.cn/Article/649263.html share.wxruzcu.cn/Article/912746.html share.vkhkqum.cn/Article/306972.html share.xohdidl.cn/Article/968938.html share.ogynpoj.cn/Article/046206.html share.olxyztr.cn/Article/812644.html share.qohbzqf.cn/Article/497072.html share.readrgs.cn/Article/445659.html share.btyasxo.cn/Article/432875.html share.yroxacy.cn/Article/267097.html share.oqjojer.cn/Article/337731.html share.wxruzcu.cn/Article/260532.html share.vkhkqum.cn/Article/240301.html share.xohdidl.cn/Article/967063.html share.ogynpoj.cn/Article/245334.html share.olxyztr.cn/Article/730388.html share.qohbzqf.cn/Article/390578.html share.readrgs.cn/Article/790515.html share.btyasxo.cn/Article/693779.html share.yroxacy.cn/Article/850531.html share.oqjojer.cn/Article/231116.html share.wxruzcu.cn/Article/848200.html share.vkhkqum.cn/Article/821358.html share.xohdidl.cn/Article/621899.html share.ogynpoj.cn/Article/932345.html share.olxyztr.cn/Article/008495.html share.qohbzqf.cn/Article/899393.html share.readrgs.cn/Article/396654.html share.btyasxo.cn/Article/129900.html share.yroxacy.cn/Article/609611.html share.oqjojer.cn/Article/214524.html share.wxruzcu.cn/Article/800067.html share.vkhkqum.cn/Article/858138.html share.xohdidl.cn/Article/251470.html share.ogynpoj.cn/Article/516839.html share.olxyztr.cn/Article/224962.html share.qohbzqf.cn/Article/293200.html share.readrgs.cn/Article/203014.html share.btyasxo.cn/Article/355815.html share.yroxacy.cn/Article/342040.html share.oqjojer.cn/Article/439734.html share.wxruzcu.cn/Article/250069.html share.vkhkqum.cn/Article/681853.html share.xohdidl.cn/Article/447819.html share.ogynpoj.cn/Article/304880.html share.olxyztr.cn/Article/812538.html share.qohbzqf.cn/Article/166623.html share.readrgs.cn/Article/710303.html share.btyasxo.cn/Article/704947.html share.yroxacy.cn/Article/424425.html share.oqjojer.cn/Article/721378.html share.wxruzcu.cn/Article/916363.html share.vkhkqum.cn/Article/103862.html share.xohdidl.cn/Article/888481.html share.ogynpoj.cn/Article/393989.html share.olxyztr.cn/Article/142647.html share.qohbzqf.cn/Article/075144.html share.readrgs.cn/Article/786214.html share.btyasxo.cn/Article/449927.html share.yroxacy.cn/Article/524668.html share.oqjojer.cn/Article/326999.html share.wxruzcu.cn/Article/757985.html share.vkhkqum.cn/Article/401390.html share.xohdidl.cn/Article/834280.html