fix fetch

1 files changed, 168 insertions, 0 deletions

diff --git a/scripts/test_improved_fetch.py b/scripts/test_improved_fetch.py
new file mode 100644
index 0000000..14490f0
--- /dev/null
+++ b/scripts/test_improved_fetch.py
@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+"""
+测试改进后的论文抓取功能
+
+验证分别查询每个类别和去重逻辑是否正常工作。
+"""
+
+import os
+import sys
+import logging
+from datetime import datetime, timezone, timedelta
+
+# 设置日志
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    handlers=[logging.StreamHandler(sys.stdout)]
+)
+logger = logging.getLogger(__name__)
+
+# Add the parent directory to the path so we can import the main module
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from scripts.fetch_papers import ArxivPaperFetcher
+
+
+def test_improved_fetching():
+    """测试改进后的抓取逻辑"""
+    
+    print("🚀 测试改进后的论文抓取逻辑")
+    print("=" * 60)
+    
+    # 创建一个模拟的fetcher（不需要OpenAI API）
+    class MockArxivFetcher(ArxivPaperFetcher):
+        def __init__(self):
+            import requests
+            self.session = requests.Session()
+            self.session.headers.update({
+                'User-Agent': 'PaperFetcher/1.0 (Test)'
+            })
+    
+    # 测试不同的时间范围
+    fetcher = MockArxivFetcher()
+    
+    print("\n🕐 测试1: 过去1天（应该显示0篇论文）")
+    end_date = datetime.now(timezone.utc)
+    start_date = end_date - timedelta(days=1)
+    papers_1day = fetcher.fetch_papers_by_date_range(start_date, end_date, max_papers=100)
+    
+    print(f"\n🕐 测试2: 过去7天（应该显示更多论文和详细分布）")
+    start_date = end_date - timedelta(days=7)
+    papers_7days = fetcher.fetch_papers_by_date_range(start_date, end_date, max_papers=300)
+    
+    print(f"\n📊 改进效果对比:")
+    print(f"   - 过去1天: {len(papers_1day)} 篇论文")
+    print(f"   - 过去7天: {len(papers_7days)} 篇论文")
+    
+    if papers_7days:
+        print(f"\n📋 论文样本 (前3篇):")
+        for i, paper in enumerate(papers_7days[:3], 1):
+            print(f"\n{i}. {paper['title'][:80]}...")
+            print(f"   arXiv ID: {paper['arxiv_id']}")
+            print(f"   更新时间: {paper['updated']}")
+            print(f"   类别: {', '.join(paper['categories'][:3])}")
+            print(f"   作者: {', '.join(paper['authors'][:2])}")
+            if len(paper['authors']) > 2:
+                print(f"         et al.")
+    
+    print(f"\n✅ 改进后的优势:")
+    print(f"   - ✅ 分别查询每个类别，避免OR查询限制")
+    print(f"   - ✅ 自动去重，避免重复论文")
+    print(f"   - ✅ 详细的类别分布统计")
+    print(f"   - ✅ 更准确的日期分布分析")
+    print(f"   - ✅ 更透明的日志显示")
+
+def test_category_overlap():
+    """测试类别重叠和去重功能"""
+    
+    print(f"\n" + "="*60)
+    print("🔍 测试类别重叠和去重功能")
+    print("="*60)
+    
+    # 简单测试：手动获取几个类别，看看重叠情况
+    import requests
+    import feedparser
+    from collections import defaultdict
+    
+    categories = ['cs.AI', 'cs.LG', 'cs.CL']
+    papers_by_category = {}
+    arxiv_ids_seen = set()
+    overlaps = defaultdict(list)
+    
+    for cat in categories:
+        print(f"\n📂 获取 {cat} 类别的论文...")
+        
+        params = {
+            'search_query': f'cat:{cat}',
+            'sortBy': 'submittedDate',
+            'sortOrder': 'descending',
+            'max_results': 50
+        }
+        
+        try:
+            response = requests.get('http://export.arxiv.org/api/query', params=params, timeout=10)
+            feed = feedparser.parse(response.content)
+            entries = feed.entries
+            
+            papers_by_category[cat] = []
+            
+            for entry in entries:
+                arxiv_id = entry.id.split('/')[-1]
+                title = entry.title.replace('\n', ' ').strip()
+                categories_list = [tag.term for tag in entry.tags] if hasattr(entry, 'tags') else []
+                
+                papers_by_category[cat].append({
+                    'arxiv_id': arxiv_id,
+                    'title': title,
+                    'categories': categories_list
+                })
+                
+                # 检查重叠
+                if arxiv_id in arxiv_ids_seen:
+                    overlaps[arxiv_id].append(cat)
+                else:
+                    arxiv_ids_seen.add(arxiv_id)
+                    overlaps[arxiv_id] = [cat]
+            
+            print(f"   获得 {len(entries)} 篇论文")
+            
+        except Exception as e:
+            print(f"   错误: {e}")
+    
+    # 分析重叠情况
+    print(f"\n📊 重叠分析:")
+    total_papers = sum(len(papers) for papers in papers_by_category.values())
+    unique_papers = len(arxiv_ids_seen)
+    duplicate_papers = total_papers - unique_papers
+    
+    print(f"   - 总获取论文: {total_papers} 篇")
+    print(f"   - 唯一论文: {unique_papers} 篇")
+    print(f"   - 重复论文: {duplicate_papers} 篇")
+    print(f"   - 去重率: {duplicate_papers/total_papers*100:.1f}%")
+    
+    # 显示一些重叠例子
+    overlap_examples = [(arxiv_id, cats) for arxiv_id, cats in overlaps.items() if len(cats) > 1][:5]
+    
+    if overlap_examples:
+        print(f"\n📋 重叠论文示例:")
+        for arxiv_id, cats in overlap_examples:
+            # 找到这篇论文的标题
+            title = "未找到标题"
+            for cat, papers in papers_by_category.items():
+                for paper in papers:
+                    if paper['arxiv_id'] == arxiv_id:
+                        title = paper['title'][:60] + "..." if len(paper['title']) > 60 else paper['title']
+                        break
+                if title != "未找到标题":
+                    break
+            
+            print(f"   - {arxiv_id}: {title}")
+            print(f"     类别: {', '.join(cats)}")
+    
+    print(f"\n✅ 这证明了去重功能的重要性！")
+
+
+if __name__ == "__main__":
+    test_improved_fetching()
+    test_category_overlap() 
\ No newline at end of file