11、长PDF文档的总结与评估策略

# 导入PyPDF2包
import PyPDF2

# 读取PDF文件内容
pdf_path = "99_data\\pdf\\2401.02385-TinyLlama.pdf"
def read_pdf(file_path):
    with open(file_path, 'rb') as file:
        reader = PyPDF2.PdfReader(file)
        text = ""
        for page in reader.pages:
            text += page.extract_text()
    return text

# 导入T5模型和分词器
from transformers import T5Tokenizer, T5ForConditionalGeneration

# 定义函数来加载模型和分词器
def load_model(model_name='t5-small'):
    tokenizer = T5Tokenizer.from_pretrained(model_name)
    model = T5ForConditionalGeneration.from_pretrained(model_name)
    return tokenizer, model

# 定义函数进行文本总结
def summarize_text(text, tokenizer, model):
    # 使用T5的前缀来指定任务类型
    text = "summarize: " + text
    
    # 对文本进行编码
    inputs = tokenizer.encode(text, return_tensors="pt", max_length=512, truncation=True)
    
    # 使用模型生成总结
    summary_ids = model.generate(inputs, max_length=150, min_length=40, length_penalty=2.0, num_beams=4, early_stopping=True)
    
    # 解码生成的总结
    summary = tokenizer.decode(summary_ids[0], skip_special_tokens=True)
    
    return summary

# 加载预训练的T5模型
tokenizer, model = load_model('t5-small')

# 对PDF内容进行总结
pdf_summary = summarize_text(pdf_text, tokenizer, model)
print("Generated Summary:", pdf_summary)

# 保存生成的摘要到文件
with open("t5_summary.txt", "w", encoding="utf-8") as file:
    file.write(t5_summary)

TinyLlama is a compact 1.1B language model pretrained on around 1 trillion tokens for approximately 3 epochs. despite its relatively small size, it significantly outperforms existing open-source language models with compara-ble sizes.

from dotenv import load_dotenv
load_dotenv()

import os 
import anthropic
os.environ["ANTHROPIC_API_KEY"] = "<your_api_key>"

def generate_summary(text):
    client = anthropic.Anthropic()
    
    system_prompt = "你是一个用于总结研究论文的AI助手。"
    
    messages = [
        {
            "role": "user", 
            "content": [
                {
                    "type": "text",
                    "text": f"""
                    请总结以下研究论文,重点关注其关键发现、方法和结论:
                    
                    {text}
                    
                    请用大约150字提供一个简明扼要的英文摘要。
                    """
                }
            ]
        }
    ]
    
    message = client.messages.create(
        model="claude-3-opus-20240229",
        max_tokens=1000,
        messages=messages,
        system=system_prompt
    )
    
    return message.content[0].text.strip()

# 使用Claude生成总结
claude_summary = generate_summary(pdf_text)
print("Summary:", claude_summary)

# 保存生成的摘要到文件
with open("claude_summary.txt", "w", encoding="utf-8") as file:
    file.write(claude_summary)

Summary:
TinyLlama is an open-source 1.1B parameter language model pretrained on approximately 1 trillion tokens for 3 epochs. It leverages advances from the open-source community to achieve better computational efficiency. Despite its small size, TinyLlama significantly outperforms existing open-source language models of comparable sizes on various downstream tasks measuring commonsense reasoning and problem-solving capabilities. The authors tracked TinyLlama's performance on benchmarks during pretraining and found that its accuracy generally improves with more compute, surpassing Pythia-1.4B. TinyLlama's pretraining code, intermediate checkpoints, and data processing details have been publicly released. The authors believe TinyLlama's compact size and strong performance make it an attractive platform for researchers to test innovative ideas related to language models. They plan to develop future versions of TinyLlama with enhanced capabilities based on the experience gained from this open research project.

# 读取摘要文件
def read_summary(file_path):
    with open(file_path, 'r', encoding='utf-8') as file:
        return file.read()
t5_summary = read_summary("t5_summary.txt")
claude_summary = read_summary("claude_summary.txt")
ref_summary = read_summary("ref_summary.txt")

from rouge import Rouge

# Rouge评估函数
def get_rouge_scores(text1, text2):
    rouge = Rouge()
    return rouge.get_scores(text1, text2)

# 示例生成摘要和参考摘要
eval_summary = "..." # 把总结结果放进来
ref_summary = "..." 把论文的原始总结放进来

# 计算ROUGE分数
rouge_scores = get_rouge_scores(eval_summary, ref_summary)

# 提取并显示关键的ROUGE指标
for metric, scores in rouge_scores[0].items():
    print(f"{metric} - Precision: {scores['p']}, Recall: {scores['r']}, F1-Score: {scores['f']}")

ROUGE Scores:
             T5 Summary  Claude Summary
Metric
rouge-1 (f)    0.568421        0.421875
rouge-1 (p)    0.843750        0.415385
rouge-1 (r)    0.428571        0.428571
rouge-2 (f)    0.466019        0.178082
rouge-2 (p)    0.727273        0.171053
rouge-2 (r)    0.342857        0.185714
rouge-l (f)    0.568421        0.406250
rouge-l (p)    0.843750        0.400000
rouge-l (r)    0.428571        0.412698

from bert_score import BERTScorer

# BERTScore评估
scorer = BERTScorer(lang="en")
P1, R1, F1_1 = scorer.score([eval_summary_1], [ref_summary])
print(f"\nSummary 1 BERTScore F1: {F1_1.tolist()[0]}")
print(f"Summary 2 BERTScore F1: {F2_2.tolist()[0]}")

T5 Summary BERTScore F1: 0.9143788814544678
Claude Summary BERTScore F1: 0.9115773439407349

# Claude评估函数
import anthropic
import re
def get_claude_score(criteria, document, summary, metric_name):
    prompt = f"""
    你将会被提供一段文章和一段摘要。你的任务是根据以下标准对摘要进行评分（1到5分）：

    评价标准：
    {criteria}

    示例：
    原文：
    {document}

    摘要：
    {summary}

    请给出{metric_name}的评分（1到5分）："""

    client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY"))
    response = client.completions.create(
        prompt=prompt,
        stop=["\n"],
        model="claude-v1",
        max_tokens=1000
    )
    result = response["choices"][0]["text"]
    score = int(re.findall(r'\d+', result)[0])
    return score

evaluation_metrics = {
    "相关性": "相关性(1-5) - 摘要是否涵盖了原文中最重要和核心的信息。",
    "准确性": "准确性(1-5) - 摘要中的信息是否准确反映了原文的内容，是否存在误导或错误信息。",
    "简洁性": "简洁性(1-5) - 摘要是否在简明扼要的基础上传达了主要信息，而不是冗长或重复。",
}

summaries = {"T5 Summary": t5_summary, "Claude Summary": claude_summary}

claude_scores = {"Metric": [], "T5 Summary": [], "Claude Summary": []}

for metric, criteria in evaluation_metrics.items():
    claude_scores["Metric"].append(metric)
    for summ_id, summ in summaries.items():
        score = get_claude_score(criteria, ref_summary, summ, metric)
        claude_scores[summ_id].append(score)

import pandas as pd
claude_df = pd.DataFrame(claude_scores).set_index("Metric")
print("\nClaude Evaluation Scores:")
print(claude_df)

Claude Evaluation Scores:
        T5 Summary  Claude Summary
Metric
相关性              3               5
准确性              5               4
简洁性              4               4