fix fetch

1 files changed, 169 insertions, 0 deletions

diff --git a/scripts/test_fetch_only.py b/scripts/test_fetch_only.py
new file mode 100644
index 0000000..db65231
--- /dev/null
+++ b/scripts/test_fetch_only.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+"""
+测试论文抓取功能 - 显示改进的日志
+
+这个脚本只测试论文抓取部分，展示分页过程和日期分布，不需要OpenAI API。
+"""
+
+import os
+import sys
+import logging
+from datetime import datetime, timezone, timedelta
+from collections import Counter
+
+# 设置日志
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    handlers=[logging.StreamHandler(sys.stdout)]
+)
+logger = logging.getLogger(__name__)
+
+# Add the parent directory to the path so we can import the main module
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from scripts.fetch_papers import ArxivPaperFetcher
+
+
+def test_paper_fetching_with_detailed_logs():
+    """测试论文抓取，显示详细的分页和日期信息"""
+    
+    print("🚀 测试改进后的论文抓取日志显示")
+    print("=" * 60)
+    
+    # 创建一个模拟的fetcher（不需要OpenAI API）
+    class MockArxivFetcher:
+        def __init__(self):
+            import requests
+            self.session = requests.Session()
+            self.session.headers.update({
+                'User-Agent': 'PaperFetcher/1.0 (Test)'
+            })
+        
+        def fetch_papers_by_date_range(self, start_date, end_date, max_papers=300):
+            """模拟我们改进后的抓取函数"""
+            logger.info(f"🔍 开始从arXiv抓取论文: {start_date.date()} 到 {end_date.date()}")
+            logger.info(f"📋 目标类别: cs.AI, cs.CL, cs.CV, cs.LG, cs.NE, cs.RO, cs.IR, cs.HC, stat.ML")
+            
+            from scripts.fetch_papers import ARXIV_BASE_URL, CS_CATEGORIES, MAX_RESULTS_PER_BATCH
+            import requests
+            import feedparser
+            
+            # Build category query
+            category_query = " OR ".join(f"cat:{cat}" for cat in CS_CATEGORIES)
+            
+            all_papers = []
+            start_index = 0
+            batch_count = 0
+            total_raw_papers = 0
+            
+            while len(all_papers) < max_papers:
+                try:
+                    batch_count += 1
+                    search_query = f"({category_query})"
+                    
+                    params = {
+                        "search_query": search_query,
+                        "sortBy": "submittedDate",
+                        "sortOrder": "descending",
+                        "start": start_index,
+                        "max_results": min(MAX_RESULTS_PER_BATCH, max_papers - len(all_papers))
+                    }
+                    
+                    logger.info(f"📦 第{batch_count}批次: 从索引{start_index}开始抓取...")
+                    
+                    response = self.session.get(ARXIV_BASE_URL, params=params, timeout=30)
+                    response.raise_for_status()
+                    
+                    feed = feedparser.parse(response.content)
+                    entries = feed.entries
+                    total_raw_papers += len(entries)
+                    
+                    logger.info(f"✅ 第{batch_count}批次获取了 {len(entries)} 篇论文")
+                    
+                    if not entries:
+                        logger.info("📭 没有更多论文可用")
+                        break
+                    
+                    # Filter papers by date and parse them
+                    batch_papers = []
+                    older_papers = 0
+                    for entry in entries:
+                        paper_date = datetime(*entry.updated_parsed[:6], tzinfo=timezone.utc)
+                        
+                        if paper_date < start_date:
+                            older_papers += 1
+                            continue
+                        
+                        if start_date <= paper_date <= end_date:
+                            paper_data = {
+                                "title": entry.title.replace('\n', ' ').strip(),
+                                "abstract": entry.summary.replace('\n', ' ').strip(),
+                                "authors": [author.name for author in entry.authors] if hasattr(entry, 'authors') else [],
+                                "published": entry.published,
+                                "updated": entry.updated,
+                                "link": entry.link,
+                                "arxiv_id": entry.id.split('/')[-1],
+                                "categories": [tag.term for tag in entry.tags] if hasattr(entry, 'tags') else []
+                            }
+                            batch_papers.append(paper_data)
+                    
+                    all_papers.extend(batch_papers)
+                    logger.info(f"📊 第{batch_count}批次筛选结果: {len(batch_papers)}篇在日期范围内, {older_papers}篇过旧")
+                    logger.info(f"📈 累计获取论文: {len(all_papers)}篇")
+                    
+                    if older_papers > 0:
+                        logger.info(f"🔚 发现{older_papers}篇超出日期范围的论文，停止抓取")
+                        break
+                    
+                    if len(entries) < MAX_RESULTS_PER_BATCH:
+                        logger.info("🔚 已达到arXiv数据末尾")
+                        break
+                    
+                    start_index += MAX_RESULTS_PER_BATCH
+                    
+                except Exception as e:
+                    logger.error(f"❌ 抓取论文时出错: {e}")
+                    break
+            
+            # 显示总结信息
+            logger.info(f"📊 抓取总结:")
+            logger.info(f"   - 总共处理了 {batch_count} 个批次")
+            logger.info(f"   - 从arXiv获取了 {total_raw_papers} 篇原始论文")
+            logger.info(f"   - 筛选出 {len(all_papers)} 篇符合日期范围的论文")
+            
+            # 显示日期分布
+            if all_papers:
+                dates = []
+                for paper in all_papers:
+                    paper_date = datetime.strptime(paper['updated'][:10], '%Y-%m-%d')
+                    dates.append(paper_date.strftime('%Y-%m-%d'))
+                
+                date_counts = Counter(dates)
+                logger.info(f"📅 论文日期分布 (前5天):")
+                for date, count in date_counts.most_common(5):
+                    days_ago = (datetime.now(timezone.utc).date() - datetime.strptime(date, '%Y-%m-%d').date()).days
+                    logger.info(f"   - {date}: {count}篇 ({days_ago}天前)")
+            
+            return all_papers
+    
+    # 测试不同的时间范围
+    fetcher = MockArxivFetcher()
+    
+    print("\n🕐 测试1: 过去1天")
+    end_date = datetime.now(timezone.utc)
+    start_date = end_date - timedelta(days=1)
+    papers_1day = fetcher.fetch_papers_by_date_range(start_date, end_date, max_papers=50)
+    
+    print(f"\n🕐 测试2: 过去7天")
+    start_date = end_date - timedelta(days=7)
+    papers_7days = fetcher.fetch_papers_by_date_range(start_date, end_date, max_papers=200)
+    
+    print(f"\n📊 对比结果:")
+    print(f"   - 过去1天: {len(papers_1day)} 篇论文")
+    print(f"   - 过去7天: {len(papers_7days)} 篇论文")
+    print(f"   - 这解释了为什么日常模式很快完成!")
+
+
+if __name__ == "__main__":
+    test_paper_fetching_with_detailed_logs() 
\ No newline at end of file