使用代理IP

星博讯 SEO推广 2026-04-09 1

处理通常涉及动态加载技术,以下是处理这类内容的方案：

使用代理IP-第1张图片-星博讯-专业SEO_网站优化技巧_搜索引擎排名提升

懒加载机制分析

百度搜索结果页的懒加载主要通过：

滚动加载 - 滚动到页面底部时加载更多结果
AJAX动态请求 - 通过异步请求获取后续内容
JavaScript渲染 - 初始页面只加载部分HTML

处理方案

方案1：模拟浏览器滚动（推荐）

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
def get_baidu_lazy_content(keyword, max_scroll=5):
    driver = webdriver.Chrome()
    driver.get(f"https://www.baidu.com/s?wd={keyword}")
    results = []
    for i in range(max_scroll):
        # 滚动到页面底部
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(2)  # 等待新内容加载
        # 获取当前所有结果
        items = driver.find_elements(By.CSS_SELECTOR, '.result.c-container.new-pmd')
        for item in items:
            if item not in results:
                results.append(item.text)
    driver.quit()
    return results

方案2：请求API接口

import requests
import json
def get_baidu_ajax_content(keyword, page=1):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
        'Referer': 'https://www.baidu.com/'
    }
    # 百度搜索结果API模式
    params = {
        'wd': keyword,
        'pn': (page-1)*10,  # 分页参数
        'rn': 10,
        'ie': 'utf-8',
        'tn': 'baidurt'
    }
    response = requests.get(
        'https://www.baidu.com/s',
        params=params,
        headers=headers
    )
    return response.text

方案3：使用无头浏览器

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
def get_full_content_headless(url):
    chrome_options = Options()
    chrome_options.add_argument("--headless")
    chrome_options.add_argument("--disable-gpu")
    driver = webdriver.Chrome(options=chrome_options)
    driver.get(url)
    # 模拟多次滚动
    last_height = driver.execute_script("return document.body.scrollHeight")
    while True:
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(2)
        new_height = driver.execute_script("return document.body.scrollHeight")
        if new_height == last_height:
            break
        last_height = new_height
    content = driver.page_source
    driver.quit()
    return content

优化建议

请求头伪装

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Connection': 'keep-alive'
}

添加延迟和代理

import random
import time
def random_delay(min_sec=1, max_sec=3):
    time.sleep(random.uniform(min_sec, max_sec))
proxies = {
    'http': 'http://your-proxy:port',
    'https': 'https://your-proxy:port'
}

错误处理和重试

from tenacity import retry, stop_after_attempt, wait_exponential
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
def safe_request(url):
    response = requests.get(url, timeout=10)
    response.raise_for_status()
    return response