初始化

begin.py

+from scrapy import cmdline
+
+cmdline.execute('scrapy crawl maintain'.split())
\ No newline at end of file

mainTainsouh/items.py

+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+
+class MaintainsouhItem(scrapy.Item):
+    # define the fields for your item here like:
+    style_id = scrapy.Field()
+    maintain_v = scrapy.Field()
+    reference_price = scrapy.Field()

mainTainsouh/middlewares.py

+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+# useful for handling different item types with a single interface
+from itemadapter import is_item, ItemAdapter
+
+
+class MaintainsouhSpiderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, or item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Request or item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
+
+
+class MaintainsouhDownloaderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)

mainTainsouh/pipelines.py

+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+
+
+# useful for handling different item types with a single interface
+import pymysql as pymysql
+from itemadapter import ItemAdapter
+
+from mainTainsouh.items import MaintainsouhItem
+
+
+class MaintainsouhPipeline:
+    def __init__(self):
+        self.host = "47.100.68.39"
+        self.port = 3310
+        self.database = "db_spider"
+        self.user = "root"
+        self.password = "dip-test"
+        self.con = pymysql.connect(host=self.host, port=self.port, database=self.database, user=self.user,
+                                   password=self.password)
+        self.cu = self.con.cursor()
+        print("实例化了数据库")
+
+    def process_item(self, item, spider):
+        if isinstance(item, MaintainsouhItem):
+            sql_q = 'select id from t_maintain_souh where souh_style_id = "{}"'.format(item['style_id'])
+            sql = 'insert into t_maintain_souh (souh_style_id, maintain, price) values ("{}","{}","{}")' \
+                .format(item['style_id'], item['maintain_v'], item['reference_price'])
+            self.con.ping(reconnect=True)
+            self.cu.execute(sql_q)
+            if str(self.cu.fetchone()) != 'None':
+                return
+            self.cu.execute(sql)
+            self.con.commit()
+
+    def __del__(self):
+        self.cu.close()
+        self.con.close()
+

mainTainsouh/settings.py

+# Scrapy settings for mainTainsouh project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://docs.scrapy.org/en/latest/topics/settings.html
+#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = 'mainTainsouh'
+
+SPIDER_MODULES = ['mainTainsouh.spiders']
+NEWSPIDER_MODULE = 'mainTainsouh.spiders'
+
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+#USER_AGENT = 'mainTainsouh (+http://www.yourdomain.com)'
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = True
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+#CONCURRENT_REQUESTS = 32
+
+# Configure a delay for requests for the same website (default: 0)
+# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+#DOWNLOAD_DELAY = 3
+# The download delay setting will honor only one of:
+#CONCURRENT_REQUESTS_PER_DOMAIN = 16
+#CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+#COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+#TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+# Override the default request headers:
+DEFAULT_REQUEST_HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                  'Chrome/99.0.4844.82 Safari/537.36'
+}
+# Enable or disable spider middlewares
+# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+#SPIDER_MIDDLEWARES = {
+#    'mainTainsouh.middlewares.MaintainsouhSpiderMiddleware': 543,
+#}
+
+# Enable or disable downloader middlewares
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#DOWNLOADER_MIDDLEWARES = {
+#    'mainTainsouh.middlewares.MaintainsouhDownloaderMiddleware': 543,
+#}
+
+# Enable or disable extensions
+# See https://docs.scrapy.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+#}
+
+# Configure item pipelines
+# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+ITEM_PIPELINES = {
+   'mainTainsouh.pipelines.MaintainsouhPipeline': 300,
+}
+LOG_LEVEL = "INFO"
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

mainTainsouh/spiders/__init__.py

+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

mainTainsouh/spiders/maintain.py

+import scrapy
+
+from mainTainsouh.items import MaintainsouhItem
+
+
+class MaintainSpider(scrapy.Spider):
+    name = 'maintain'
+    allowed_domains = ['auto.sohu.com']
+    start_urls = ['http://db.auto.sohu.com/home/']
+
+    def parse(self, response, **kwargs):
+        all_brand_li = response.xpath('//*[@id="treeNav"]/ul/li[@class="close_child"]')
+        for i in all_brand_li:
+            all_maker_li = i.xpath("ul")
+            for j in all_maker_li:
+                all_series_li = j.xpath('li/a[@class="model-a"]')
+                for k in all_series_li:
+                    # 获取车型保养信息地址
+                    maintain_url = 'http:' + k.attrib.get('href') + '/maintenance.html'
+                    yield scrapy.Request(maintain_url, callback=self.parse_maintain)
+
+    def parse_maintain(self, response, **kwargs):
+        all_maintain_tb = response.xpath('//*[@id="type_tb_L"]/div')
+        # 遍历车系下所有车款
+        for i in all_maintain_tb:
+            maintain = MaintainsouhItem()
+            maintain['style_id'] = i.attrib.get('id')[4:-2]
+            value = []
+            all_item = i.xpath('table[1]/thead/tr/th')
+            all_item_v = i.xpath('table[1]/tbody/tr')
+            # 遍历所有项目item
+            for j in range(1, 11):
+                item1 = all_item[j].xpath('text()')
+                #  拼接item内容并去掉非数字文字字母字符
+                item = ''
+                if len(item1) > 1:
+                    item = (str(item1[0].root) + "" + str(item1[1].root)).strip()
+                else:
+                    item = str(item1[0].root)
+
+                # 遍历所有里程
+                item_list = []
+                for k in all_item_v:
+                    item_v = k.xpath('td')
+                    item_list.append({"mile": item_v[0].xpath('text()')[0].root,
+                                      "level": "2" if item_v[j].xpath('text()')[0].root == "●" else "0"})
+                value.append({"item": item, "items": item_list})
+            maintain['maintain_v'] = str(value)
+            all_price = i.xpath('table[2]/tbody/tr')
+            price = []
+            for l in all_price:
+                price_dict = {}
+                td = l.xpath('td')
+                price.append(
+                    {"item": '' if len(td[0].xpath('text()')) == 0 else str(td[0].xpath('text()')[0].root).strip(),
+                     "price": '' if len(td[1].xpath('text()')) == 0 else str(td[1].xpath('text()')[0].root).strip(),
+                     "other": '' if len(td[2].xpath('text()')) == 0 else str(td[2].xpath('text()')[0].root).strip()})
+                price.append(
+                    {"item": '' if len(td[3].xpath('text()')) == 0 else str(td[3].xpath('text()')[0].root).strip(),
+                     "price": '' if len(td[4].xpath('text()')) == 0 else str(td[4].xpath('text()')[0].root).strip(),
+                     "other": '' if len(td[5].xpath('text()')) == 0 else str(td[5].xpath('text()')[0].root).strip()})
+            maintain['reference_price'] = str(price[0:-1])
+            yield maintain

scrapy.cfg

+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = mainTainsouh.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = mainTainsouh