附录E：全栈代码示例库

#!/usr/bin/env python3
"""
Nginx日志分析脚本：提取搜索引擎爬虫行为。
支持日志格式：combined (默认Nginx格式)
使用方法：python3 analyze_nginx_logs.py /var/log/nginx/access.log
"""

import re
import sys
import json
from collections import defaultdict, Counter
from datetime import datetime

# 配置：要监控的爬虫User-Agent关键字
TARGET_CRAWLERS = [
    'Googlebot',
    'GPTBot',
    'Bingbot',
    'Baiduspider',
    'Bytespider',
    'ClaudeBot',
    'CCBot',
    'DeepSeek-Bot',
    'YandexBot',
    'Applebot',
]

# 日志正则（Nginx combined格式）
LOG_PATTERN = re.compile(
    r'(?P<ip>\S+) - (?P<user>\S+) \[(?P<time>[^\]]+)\] '
    r'"(?P<method>\S+) (?P<path>\S+) (?P<protocol>\S+)" '
    r'(?P<status>\d{3}) (?P<size>\S+) "(?P<referer>[^"]*)" '
    r'"(?P<ua>[^"]*)"'
)

def parse_log_line(line):
    """解析单行日志，返回字典或None"""
    match = LOG_PATTERN.match(line)
    if not match:
        return None
    return match.groupdict()

def identify_crawler(ua_string):
    """根据User-Agent识别爬虫类型"""
    for crawler in TARGET_CRAWLERS:
        if crawler.lower() in ua_string.lower():
            return crawler
    return None

def analyze_logs(log_file_path):
    """主分析函数"""
    crawler_stats = defaultdict(lambda: {
        'total_requests': 0,
        'status_codes': Counter(),
        'urls': Counter(),
        'first_seen': None,
        'last_seen': None,
    })

    with open(log_file_path, 'r', encoding='utf-8', errors='ignore') as f:
        for line in f:
            parsed = parse_log_line(line)
            if not parsed:
                continue

            ua = parsed['ua']
            crawler = identify_crawler(ua)
            if not crawler:
                continue

            stats = crawler_stats[crawler]
            stats['total_requests'] += 1
            stats['status_codes'][parsed['status']] += 1
            stats['urls'][parsed['path']] += 1

            # 记录时间
            try:
                log_time_str = parsed['time'].split(' ')[0]  # 去除时区
                log_time = datetime.strptime(log_time_str, '%d/%b/%Y:%H:%M:%S')
                if stats['first_seen'] is None or log_time < stats['first_seen']:
                    stats['first_seen'] = log_time
                if stats['last_seen'] is None or log_time > stats['last_seen']:
                    stats['last_seen'] = log_time
            except ValueError:
                pass

    # 格式化输出
    result = {}
    for crawler, stats in crawler_stats.items():
        result[crawler] = {
            'total_requests': stats['total_requests'],
            'unique_urls': len(stats['urls']),
            'status_code_distribution': dict(stats['status_codes'].most_common()),
            'top_urls': [{'url': url, 'count': cnt} for url, cnt in stats['urls'].most_common(10)],
            'first_seen': stats['first_seen'].isoformat() if stats['first_seen'] else None,
            'last_seen': stats['last_seen'].isoformat() if stats['last_seen'] else None,
        }

    return result

if __name__ == '__main__':
    if len(sys.argv) != 2:
        print("用法: python3 analyze_nginx_logs.py <log_file_path>")
        sys.exit(1)

    log_path = sys.argv[1]
    print(f"正在分析日志文件: {log_path}")
    analysis_result = analyze_logs(log_path)

    # 输出JSON结果
    output_json = json.dumps(analysis_result, indent=2, ensure_ascii=False)
    print("\n===== 爬虫分析报告 =====")
    print(output_json)

    # 可选：保存到文件
    with open('crawler_analysis_report.json', 'w', encoding='utf-8') as f:
        f.write(output_json)
    print("\n报告已保存至 crawler_analysis_report.json")