每日Skill推荐：scrape-web 网页抓取神器#

1
from scrapling.fetchers import Fetcher
2

3
page = Fetcher.get('https://quotes.toscrape.com/')
4
quotes = page.css('.quote .text::text').getall()
5
for q in quotes:
6
    print(q)

1
from scrapling.fetchers import FetcherSession
2

3
with FetcherSession(impersonate='chrome') as session:
4
    page = session.get('https://example.com/', stealthy_headers=True)
5
    links = page.css('a::attr(href)').getall()

1
from scrapling.fetchers import DynamicFetcher
2

3
page = DynamicFetcher.fetch(
4
    'https://example.com',
5
    wait_selector=('.results', 'visible'),
6
    network_idle=True,
7
)

1
from scrapling.fetchers import StealthyFetcher
2

3
page = StealthyFetcher.fetch(
4
    'https://protected-site.com',
5
    headless=True,
6
    solve_cloudflare=True,
7
    block_webrtc=True,
8
    hide_canvas=True,
9
)

1
page.css('h1::text').get()                    # CSS文本
2
page.css('a::attr(href)').getall()            # CSS属性
3
page.xpath('//div[@class="x"]/text()').getall()  # XPath
4
page.find_by_text('Read more', tag='a')       # 按文本查找
5
page.find_by_regex(r'\$\d+\.\d{2}')            # 按正则
6
first_product.css('.product')[0].find_similar()  # 相似元素

1
from scrapling.spiders import Spider, Response
2

3
class MySpider(Spider):
4
    name = "my_spider"
5
    start_urls = ["https://example.com/"]
6

7
    async def parse(self, response: Response):
8
        for item in response.css('.item'):
9
            yield {
10
                "title": item.css('h2::text').get(),
11
                "link": item.css('a::attr(href)').get(),
12
            }
13
        next_page = response.css('.next a::attr(href)').get()
14
        if next_page:
15
            yield response.follow(next_page)
16

17
result = MySpider().start()
18
result.items.to_json("items.json")