Empirical Analysis of FOMC Communications Using RAG and Small Language Models

# Install required packages if needed.
# On the shared hub, many of these may already be installed.
# Uncomment only if your environment is missing them.

# !pip install -q llama-cpp-python sentence-transformers pandas matplotlib seaborn scikit-learn

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from datetime import datetime
import re
from IPython.display import Image, display
from io import StringIO

sns.set_style('whitegrid')
plt.rcParams['figure.figsize'] = (12, 5)

print("✓ Libraries loaded")

# Load FOMC corpus from GitHub
FED_CORPUS_URL = "https://raw.githubusercontent.com/vtasca/fed-statement-scraping/refs/heads/master/communications.csv"

print("Loading FOMC corpus...")
df = pd.read_csv(FED_CORPUS_URL)
df['Date'] = pd.to_datetime(df['Date'])
df['Release Date'] = pd.to_datetime(df['Release Date'])

print(f"✓ Loaded {len(df):,} documents ({df['Date'].min().date()} to {df['Date'].max().date()})")

# Complete historical FOMC decisions dataset (CSV format for reproducibility)
# Source: Federal Reserve Board - Historical FOMC Meeting Dates and Policy Actions
# https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm

historical_data_csv = """Date,Policy_Outcome,Change_BP,Target_Range_Lower,Target_Range_Upper
2026-01-28,Hold,0,3.50,3.75
2025-12-10,Lower,-25,3.50,3.75
2025-10-29,Lower,-25,3.75,4.00
2025-09-17,Lower,-50,4.00,4.25
2025-07-30,Hold,0,4.50,4.75
2025-06-11,Hold,0,4.50,4.75
2025-04-29,Hold,0,4.50,4.75
2025-03-18,Hold,0,4.50,4.75
2025-01-28,Hold,0,4.50,4.75
2024-12-17,Lower,-25,4.50,4.75
2024-11-06,Lower,-25,4.75,5.00
2024-09-17,Lower,-50,5.00,5.25
2024-07-30,Hold,0,5.50,5.75
2024-06-11,Hold,0,5.50,5.75
2024-04-30,Hold,0,5.50,5.75
2024-03-19,Hold,0,5.50,5.75
2024-01-30,Hold,0,5.50,5.75
2023-12-12,Hold,0,5.50,5.75
2023-10-31,Hold,0,5.50,5.75
2023-09-19,Hold,0,5.50,5.75
2023-07-25,Raise,25,5.50,5.75
2023-06-13,Hold,0,5.25,5.50
2023-05-02,Raise,25,5.25,5.50
2023-03-21,Raise,25,5.00,5.25
2023-01-31,Raise,25,4.75,5.00
2022-12-13,Raise,50,4.50,4.75
2022-11-01,Raise,75,4.00,4.25
2022-09-20,Raise,75,3.25,3.50
2022-07-26,Raise,75,2.50,2.75
2022-06-14,Raise,75,1.75,2.00
2022-05-03,Raise,50,1.00,1.25
2022-03-15,Raise,25,0.50,0.75
2022-01-25,Hold,0,0.25,0.50"""

# Load into DataFrame
ground_truth = pd.read_csv(StringIO(historical_data_csv))
ground_truth['Date'] = pd.to_datetime(ground_truth['Date'])

print(f"✓ Loaded {len(ground_truth)} ground truth policy decisions")
print(f"  Coverage: {ground_truth['Date'].min().date()} to {ground_truth['Date'].max().date()}")
print(f"\nPolicy Outcome Distribution:")
print(ground_truth['Policy_Outcome'].value_counts())

# Merge ground truth with statements using nearest-date matching.
# FOMC statements in the corpus are sometimes dated 1-2 days off from the
# official meeting end-date used in the ground-truth table, so an exact join
# drops most rows.  merge_asof with a 3-day tolerance fixes this without
# risk of cross-meeting contamination (meetings are ≥6 weeks apart).

statements = df[df['Type'] == 'Statement'].copy()
statements = statements.sort_values('Date').reset_index(drop=True)
ground_truth_sorted = ground_truth.sort_values('Date').reset_index(drop=True)

# Diagnostic: show how corpus dates align with ground-truth dates
recent_gt = ground_truth_sorted[ground_truth_sorted['Date'] >= '2024-01-01']
recent_st = statements[statements['Date'] >= '2024-01-01'][['Date']]
print("Date alignment check (2024+ meetings):")
print(f"  Ground-truth dates : {sorted(recent_gt['Date'].dt.date.tolist())}")
print(f"  Corpus stmt dates  : {sorted(recent_st['Date'].dt.date.tolist())}\n")

statements = pd.merge_asof(
    statements,
    ground_truth_sorted,
    on='Date',
    direction='nearest',
    tolerance=pd.Timedelta('3 days')
)

# Keep only statements that matched a ground-truth entry
statements_labeled = statements.dropna(subset=['Policy_Outcome']).copy()

print(f"✓ Matched {len(statements_labeled)} statements with ground truth labels")
print(f"  Unmatched statements: {len(statements) - len(statements_labeled)}")

# Visualize
fig, ax = plt.subplots(figsize=(14, 5))
yearly = statements_labeled.groupby(
    [statements_labeled['Date'].dt.year, 'Policy_Outcome']
).size().unstack(fill_value=0)

yearly.plot(kind='bar', stacked=True, ax=ax, color=['#C1666B', '#48A9A6', '#4357AD'])
ax.set_xlabel('Year', fontsize=11)
ax.set_ylabel('Number of Meetings', fontsize=11)
ax.set_title('FOMC Policy Decisions (Ground Truth)', fontsize=13, fontweight='bold')
ax.legend(title='Decision')
ax.grid(axis='y', alpha=0.3)
plt.xticks(rotation=0)
plt.tight_layout()
plt.show()

url = "https://raw.githubusercontent.com/ds-modules/Small_Models_SP26/main/lance/fredgraph.png"
display(Image(url=url))

'''
Description:
This chart shows the U.S. Federal Funds Effective Rate from 2021 to early 2026. Rates remained near 0% through early 2022, rose sharply throughout 2022–2023 to above 5% as the Federal Reserve tightened monetary policy to combat inflation, and then gradually declined through 2024–2025.

Explore the data:
https://fred.stlouisfed.org/series/FEDFUNDS
'''

def chunk_text(text, chunk_size=500, overlap=25):
    """Split text into overlapping chunks."""
    words = text.split()
    return [' '.join(words[i:i + chunk_size]) for i in range(0, len(words), chunk_size - overlap)]

# Build chunked corpus using itertuples (faster than iterrows)
chunked_corpus = []
for row in df.itertuples():
    for i, chunk in enumerate(chunk_text(row.Text)):
        chunked_corpus.append({
            'text': chunk,
            'date': row.Date,
            'doc_type': row.Type,
            'chunk_id': f"{row.Date.date()}_{row.Type}_{i}"
        })

print(f"✓ Created {len(chunked_corpus):,} chunks")

from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity

print("Loading embedding model...")
embedding_model = SentenceTransformer('paraphrase-MiniLM-L3-v2')
print(f"✓ Loaded (dim={embedding_model.get_sentence_embedding_dimension()})")

print("Computing embeddings...")
corpus_texts = [chunk['text'] for chunk in chunked_corpus]
corpus_embeddings = embedding_model.encode(corpus_texts, batch_size=64, show_progress_bar=True)
print(f"✓ Embeddings: {corpus_embeddings.shape}")

class VectorIndex:
    """Vector search index with pre-normalized embeddings for fast cosine similarity."""
    def __init__(self, embeddings, metadata):
        norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
        self.embeddings = embeddings / norms  # normalize once at build time
        self.metadata = metadata

    def search(self, query_embedding, top_k=5):
        query_norm = query_embedding / np.linalg.norm(query_embedding)
        similarities = self.embeddings @ query_norm          # plain matmul, no norm recomputation
        top_k_idx = np.argpartition(similarities, -top_k)[-top_k:]  # O(n) vs O(n log n)
        top_indices = top_k_idx[np.argsort(similarities[top_k_idx])[::-1]]
        return [{'score': float(similarities[idx]), 'metadata': self.metadata[idx]} for idx in top_indices]

vector_index = VectorIndex(corpus_embeddings, chunked_corpus)
print("✓ Vector index built")

def keyword_search(query, corpus, top_k=5):
    """Simple keyword-based retrieval."""
    keywords = query.lower().split()
    scores = np.array([sum(chunk['text'].lower().count(kw) for kw in keywords) for chunk in corpus])
    top_k_idx = np.argpartition(scores, -top_k)[-top_k:]  # O(n) vs O(n log n)
    top_indices = top_k_idx[np.argsort(scores[top_k_idx])[::-1]]
    return [{'score': int(scores[idx]), 'metadata': corpus[idx]} for idx in top_indices]

print("✓ Keyword search ready")

!ls /home/jovyan/shared

from llama_cpp import Llama
import torch, os

model_path = "/home/jovyan/shared/qwen2-1_5b-instruct-q4_0.gguf"
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Device: {device}")

# Configure GPU layers — offload all layers to GPU if available, else CPU only
n_gpu_layers = -1 if device == "cuda" else 0

# Set threads for CPU inference
n_threads = os.cpu_count() or 4

print(f"Loading {model_path}...")
llm = Llama(
    model_path=model_path,
    n_ctx=4096,
    n_gpu_layers=n_gpu_layers,
    n_threads=n_threads,
    verbose=False,
)

label = "GPU (CUDA)" if device == "cuda" else "CPU"
print(f"✓ Model loaded ({label}, Q4_0 quantized)")

RAG_PROMPT = """You are analyzing FOMC communications. Answer using ONLY the excerpts below.
RULES:
1. Use ONLY information from excerpts
2. Cite: (FOMC, YYYY-MM-DD, statement/minutes)
3. If insufficient info, say: "Insufficient information"
4. No outside knowledge
EXCERPTS:
{excerpts}
QUESTION: {question}
ANSWER:"""

def format_chunks(results):
    return "\n\n".join(
        f"[{i}] (FOMC, {r['metadata']['date'].date()}, {r['metadata']['doc_type']})\n{r['metadata']['text']}"
        for i, r in enumerate(results, 1)
    )

def generate(prompt, max_tokens=100):
    """Generate response using llama-cpp. Greedy decoding for deterministic, fast inference."""
    response = llm.create_chat_completion(
        messages=[{"role": "user", "content": prompt}],
        max_tokens=max_tokens,
        temperature=0.0,  # greedy / deterministic
    )
    return response["choices"][0]["message"]["content"].strip()

def generate_with_rag(question, retrieved_chunks):
    """Generate with RAG."""
    prompt = RAG_PROMPT.format(excerpts=format_chunks(retrieved_chunks), question=question)
    return generate(prompt)

def generate_no_retrieval(question):
    """Generate without retrieval."""
    return generate(f"Answer this question about FOMC communications: {question}")

print("✓ Generation functions ready")

# Corpus-verified evaluation questions
eval_questions = [
    {
        'id': 1,
        'question': 'What target range did the Committee maintain in January 2026?',
        'expected': '3-1/2 to 3-3/4 percent',
        'pattern': r'3[\s-]*1/2.*3[\s-]*3/4',
        'verifiable': True
    },
    {
        'id': 2,
        'question': 'By how much did the Committee lower the target range in December 2025?',
        'expected': '1/4 percentage point',
        'pattern': r'1/4.*percentage point|25.*basis points',
        'verifiable': True
    },
    {
        'id': 3,
        'question': 'How did the Committee describe inflation in December 2025?',
        'expected': 'somewhat elevated',
        'pattern': r'somewhat elevated',
        'verifiable': True
    },
    {
        'id': 4,
        'question': 'Who voted against the January 2026 action?',
        'expected': 'Stephen I. Miran and Christopher J. Waller',
        'pattern': r'Miran.*Waller|Waller.*Miran',
        'verifiable': True
    },
    {
        'id': 5,
        'question': 'What balance sheet action did the Committee announce in December 2025?',
        'expected': 'initiate purchases of shorter-term Treasury securities',
        'pattern': r'purchases.*Treasury.*securities|Treasury.*purchases',
        'verifiable': True
    },
    {
        'id': 6,
        'question': 'What is the Fed\'s GDP growth forecast for 2027?',
        'expected': 'UNANSWERABLE',
        'pattern': None,
        'verifiable': False
    }
]

print(f"Evaluation set: {len(eval_questions)} questions")
print(f"  Corpus-extractable: {sum(q['verifiable'] for q in eval_questions)}")
print(f"  Refusal tests: {sum(not q['verifiable'] for q in eval_questions)}")

comparison_results = []

print("Running baseline vs RAG comparison...\n")

for q in eval_questions:
    print(f"Q{q['id']}: {q['question'][:60]}...")

    # Encode query once; shared across all retrieval methods
    query_emb = embedding_model.encode(q['question'])

    # Method 1: No retrieval
    no_ret_answer = generate_no_retrieval(q['question'])

    # Method 2: Keyword retrieval
    kw_retrieved = keyword_search(q['question'], chunked_corpus, top_k=3)
    kw_answer = generate_with_rag(q['question'], kw_retrieved)

    # Method 3: Semantic RAG
    sem_retrieved = vector_index.search(query_emb, top_k=3)
    sem_answer = generate_with_rag(q['question'], sem_retrieved)

    comparison_results.append({
        'id': q['id'],
        'question': q['question'],
        'expected': q['expected'],
        'pattern': q['pattern'],
        'verifiable': q['verifiable'],
        'no_ret_answer': no_ret_answer,
        'kw_answer': kw_answer,
        'sem_answer': sem_answer,
        'kw_score': kw_retrieved[0]['score'] if kw_retrieved else 0,
        'sem_score': sem_retrieved[0]['score']
    })

    print(f"  ✓ Generated 3 responses\n")

print("✓ All comparisons complete")

def check_citation(answer):
    """Check for proper citation format."""
    pattern = r'\(FOMC,\s*\d{4}-\d{2}-\d{2},\s*(statement|minutes|Statement|Minute)\)'
    return bool(re.search(pattern, answer))

def check_refusal(answer):
    """Check if model properly refused."""
    patterns = ['insufficient', 'cannot answer', 'not available', 'do not have', 'not in']
    return any(p in answer.lower() for p in patterns)

def grade_answer(answer, expected, pattern, verifiable):
    """Grade answer with regex pattern matching."""
    # Check citation
    has_citation = check_citation(answer)

    # Check correctness
    if not verifiable:
        # Should refuse
        correct = check_refusal(answer)
        grounded = correct
    else:
        # Should extract fact
        if pattern:
            # Use regex pattern
            correct = bool(re.search(pattern, answer, re.IGNORECASE))
        else:
            # Fallback: check if expected appears
            correct = expected.lower() in answer.lower()

        grounded = has_citation

    return {
        'correct': correct,
        'citation': has_citation,
        'grounded': grounded
    }

# Grade all answers for all methods
for r in comparison_results:
    r['no_ret_grade'] = grade_answer(r['no_ret_answer'], r['expected'], r['pattern'], r['verifiable'])
    r['kw_grade'] = grade_answer(r['kw_answer'], r['expected'], r['pattern'], r['verifiable'])
    r['sem_grade'] = grade_answer(r['sem_answer'], r['expected'], r['pattern'], r['verifiable'])

print("✓ Grading complete")

# Build comparison table
comparison_table = []

for r in comparison_results:
    comparison_table.append({
        'ID': r['id'],
        'Question': r['question'][:45] + '...',
        'No_Ret_Correct': '✓' if r['no_ret_grade']['correct'] else '✗',
        'No_Ret_Citation': '✓' if r['no_ret_grade']['citation'] else '✗',
        'KW_Correct': '✓' if r['kw_grade']['correct'] else '✗',
        'KW_Citation': '✓' if r['kw_grade']['citation'] else '✗',
        'RAG_Correct': '✓' if r['sem_grade']['correct'] else '✗',
        'RAG_Citation': '✓' if r['sem_grade']['citation'] else '✗',
        'RAG_Score': f"{r['sem_score']:.3f}"
    })

comp_df = pd.DataFrame(comparison_table)

print("BASELINE vs RAG COMPARISON")
print("=" * 120)
print(comp_df.to_string(index=False))
print("=" * 120)

def compute_metrics(col_prefix):
    n = len(comp_df)
    acc  = (comp_df[f'{col_prefix}_Correct']  == '✓').sum() / n * 100
    cite = (comp_df[f'{col_prefix}_Citation'] == '✓').sum() / n * 100
    return {'Accuracy': acc, 'Citation': cite}

metrics_summary = pd.DataFrame({
    'No Retrieval': compute_metrics('No_Ret'),
    'Keyword':      compute_metrics('KW'),
    'Semantic RAG': compute_metrics('RAG'),
}).T

print("\nMETRICS BY METHOD")
print("=" * 50)
print(metrics_summary.to_string(float_format=lambda x: f"{x:.1f}%"))
print("=" * 50)

import difflib

def sentence_split(text):
    return [s.strip() for s in re.split(r'(?<=[.!?])\s+', text) if s.strip()]

# Get two most recent consecutive statements
sorted_stmts = statements_labeled.sort_values('Date').reset_index(drop=True)
current  = sorted_stmts.iloc[-1]
previous = sorted_stmts.iloc[-2]

# Use difflib to identify textual changes between statements
curr_sents = sentence_split(current['Text'])
prev_sents = sentence_split(previous['Text'])
diff    = list(difflib.Differ().compare(prev_sents, curr_sents))
added   = [line[2:] for line in diff if line.startswith('+ ')]
removed = [line[2:] for line in diff if line.startswith('- ')]

print(f"Actual changes: {len(added)} added, {len(removed)} removed")
print(f"Policy change: {previous['Policy_Outcome']} → {current['Policy_Outcome']}\n")

# Test 1: Identify changes
diff_prompt_1 = f"""Compare these FOMC statements and list the key textual changes:

PREVIOUS ({previous['Date'].date()}):
{previous['Text'][:500]}...

CURRENT ({current['Date'].date()}):
{current['Text'][:500]}...

What specific language changed?"""

diff_answer_1 = generate(diff_prompt_1, max_tokens=120)

print("DIFFERENCING TEST 1: Identify Changes")
print("-" * 80)
print(f"Model response:\n{diff_answer_1}\n")

# Test 2: Policy stance
diff_prompt_2 = f"""Did the policy stance become more hawkish or dovish from {previous['Date'].date()} to {current['Date'].date()}?

Context: Previous decision was {previous['Policy_Outcome']}, current is {current['Policy_Outcome']}."""

diff_answer_2 = generate(diff_prompt_2, max_tokens=80)

print("DIFFERENCING TEST 2: Policy Stance")
print("-" * 80)
print(f"Model response:\n{diff_answer_2}\n")

# Score differencing performance
def score_differencing(answer, added_sents, removed_sents, policy_change):
    """Score differencing answer."""
    answer_lower = answer.lower()

    # Check if identified additions
    identified_add = any(s[:30].lower() in answer_lower for s in added_sents[:3])

    # Check if identified removals
    identified_remove = any(s[:30].lower() in answer_lower for s in removed_sents[:3])

    # Check if linked to policy change
    linked_policy = policy_change.lower() in answer_lower

    return {
        'identified_additions': identified_add,
        'identified_removals': identified_remove,
        'linked_to_policy': linked_policy,
        'score': sum([identified_add, identified_remove, linked_policy])
    }

diff_score = score_differencing(diff_answer_1, added, removed, current['Policy_Outcome'])

print("DIFFERENCING PERFORMANCE SCORE")
print("=" * 50)
print(f"Identified additions: {'✓' if diff_score['identified_additions'] else '✗'}")
print(f"Identified removals: {'✓' if diff_score['identified_removals'] else '✗'}")
print(f"Linked to policy: {'✓' if diff_score['linked_to_policy'] else '✗'}")
print(f"Overall score: {diff_score['score']}/3")
print("=" * 50)

def normalize_prediction(text):
    """Map raw model output to Raise / Hold / Lower."""
    t = text.lower()
    if any(w in t for w in ['raise', 'increas']):
        return 'Raise'
    if any(w in t for w in ['lower', 'cut', 'decreas', 'reduc']):
        return 'Lower'
    if any(w in t for w in ['hold', 'maintain', 'unchanged']):
        return 'Hold'
    return 'Unknown'

# Sample 10 statements for classification
test_statements = statements_labeled.sample(n=10, random_state=42).reset_index(drop=True)

classification_results = []

print("Running policy classification...\n")

for idx, row in test_statements.iterrows():
    prompt = f"""Classify the FOMC policy decision as: Raise, Hold, or Lower.

Statement excerpt:
{row['Text'][:400]}

Decision (one word):"""

    prediction = generate(prompt, max_tokens=10)
    pred_class = normalize_prediction(prediction)
    correct    = (pred_class == row['Policy_Outcome'])

    classification_results.append({
        'date':      row['Date'].date(),
        'true':      row['Policy_Outcome'],
        'predicted': pred_class,
        'correct':   correct
    })

    if idx < 5:
        print(f"{row['Date'].date()}: {row['Policy_Outcome']} → {pred_class} {'✓' if correct else '✗'}")

print("\n✓ Classification complete")

class_df = pd.DataFrame(classification_results)

print("\nPOLICY OUTCOME CLASSIFICATION RESULTS")
print("=" * 60)
print(class_df.to_string(index=False))

accuracy = class_df['correct'].mean() * 100
print(f"\nClassification Accuracy: {accuracy:.1f}%")
print("=" * 60)

# Confusion matrix
from sklearn.metrics import confusion_matrix

valid = class_df[class_df['predicted'] != 'Unknown']

if len(valid) > 0:
    cm = confusion_matrix(valid['true'], valid['predicted'], labels=['Raise', 'Hold', 'Lower'])

    fig, ax = plt.subplots(figsize=(8, 6))
    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
                xticklabels=['Raise', 'Hold', 'Lower'],
                yticklabels=['Raise', 'Hold', 'Lower'], ax=ax)
    ax.set_xlabel('Predicted')
    ax.set_ylabel('True')
    ax.set_title('Policy Outcome Confusion Matrix')
    plt.tight_layout()
    plt.show()

# Reuse compute_metrics for all three methods
no_ret = compute_metrics('No_Ret')
kw     = compute_metrics('KW')
rag    = compute_metrics('RAG')

policy_acc = class_df['correct'].mean() * 100

print("FINAL EXPERIMENTAL RESULTS")
print("=" * 80)
model_label = "Q4_0 with CUDA offload" if device == "cuda" else "Q4_0 CPU"
print(f"Model: Qwen2.5-1.5B-Instruct ({model_label})")
print(f"Corpus: {len(df)} FOMC documents ({df['Date'].min().year}-{df['Date'].max().year})")
print(f"Ground truth: {len(ground_truth)} labeled policy decisions")
print(f"\nQA ACCURACY (6 questions):")
print(f"  No Retrieval:    {no_ret['Accuracy']:.1f}%")
print(f"  Keyword:         {kw['Accuracy']:.1f}%")
print(f"  Semantic RAG:    {rag['Accuracy']:.1f}%")
print(f"\nCITATION RATE:")
print(f"  No Retrieval:    {no_ret['Citation']:.1f}%")
print(f"  Keyword:         {kw['Citation']:.1f}%")
print(f"  Semantic RAG:    {rag['Citation']:.1f}%")
print(f"\nPOLICY CLASSIFICATION (10 statements):")
print(f"  Accuracy:        {policy_acc:.1f}%")
print(f"\nDIFFERENCING PERFORMANCE:")
print(f"  Score:           {diff_score['score']}/3")
print("=" * 80)