Empirical Analysis of FOMC Communications Using RAG and Small Language Models

# Install required packages if needed.
# On the shared hub, many of these may already be installed.
# Uncomment only if your environment is missing them.

# !pip install -q llama-cpp-python sentence-transformers pandas matplotlib seaborn scikit-learn

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from datetime import datetime
import re
from IPython.display import Image, display
from io import StringIO

sns.set_style('whitegrid')
plt.rcParams['figure.figsize'] = (12, 5)

print("✓ Libraries loaded")

# Load FOMC corpus from GitHub
FED_CORPUS_URL = "https://raw.githubusercontent.com/vtasca/fed-statement-scraping/refs/heads/master/communications.csv"

print("Loading FOMC corpus...")
df = pd.read_csv(FED_CORPUS_URL)
df['Date'] = pd.to_datetime(df['Date'])
df['Release Date'] = pd.to_datetime(df['Release Date'])

print(f"✓ Loaded {len(df):,} documents ({df['Date'].min().date()} to {df['Date'].max().date()})")

# Load the federal funds target rate lower limit directly from FRED
# DFEDTARL = Federal Funds Target Range - Lower Limit
# cosd = start date, coed = end date
fred_url = "https://fred.stlouisfed.org/graph/fredgraph.csv?id=DFEDTARL&cosd=2000-01-01&coed=2026-03-08"
print("Loading FOMC target rate data from FRED...")
df_rate = pd.read_csv(fred_url)
df_rate['Date'] = pd.to_datetime(df_rate['observation_date'])
print(f"✓ Loaded {len(df_rate):,} daily rate observations")
df_rate.tail()

# Extract only 'Statement' type documents from the corpus
# Filter to 2008-2026 — this is when the target rate range format was introduced
statements_panel = (
    df[df['Type'] == 'Statement']
    .copy()
    .sort_values('Date')
    .reset_index(drop=True)
)

statements_panel = statements_panel[
    (statements_panel['Date'] >= pd.Timestamp('2008-12-15'))
    & (statements_panel['Date'] <= pd.Timestamp('2026-12-31'))
].reset_index(drop=True)

print(
    f"Statements available: {len(statements_panel)}"
    f" ({statements_panel['Date'].min().date()} to {statements_panel['Date'].max().date()})"
)
statements_panel.head()

# Clean dates for both dataframes before merging
statements_panel["Date"] = pd.to_datetime(statements_panel["Date"])
df_rate["Date"] = pd.to_datetime(df_rate["Date"])

# Keep only the date and rate columns from the FRED data
fed_target_history = (
    df_rate[["Date", "DFEDTARL"]]
    .dropna()
    .sort_values("Date")
    .reset_index(drop=True)
)

# Match each statement to the most recent available target rate
# merge_asof joins on the nearest past date (backward lookup)
statements_with_rate = pd.merge_asof(
    statements_panel.sort_values("Date"),
    fed_target_history,
    on="Date",
    direction="backward"
)

# Compute meeting-to-meeting policy change (delta = rate difference from previous meeting)
statements_with_rate["delta"] = statements_with_rate["DFEDTARL"].diff()

# Label the policy move based on whether the rate went up, down, or stayed the same
# Raise = rate increased, Lower = rate decreased, Hold = no change
for row_idx in range(len(statements_with_rate)):
    delta_val = statements_with_rate.loc[row_idx, "delta"]
    if delta_val > 0:
        statements_with_rate.loc[row_idx, "Policy_Outcome"] = "Raise"
    elif delta_val < 0:
        statements_with_rate.loc[row_idx, "Policy_Outcome"] = "Lower"
    else:
        statements_with_rate.loc[row_idx, "Policy_Outcome"] = "Hold"

# The first meeting has no previous meeting to compare — remove it
statements_with_rate.loc[statements_with_rate["delta"].isna(), "Policy_Outcome"] = None

statements_with_rate.head()

# Remove rows where we could not compute a rate change (the very first meeting)
statement_policy_panel = statements_with_rate.dropna(subset=["DFEDTARL", "delta"]).reset_index(drop=True)

# Create the labeled statements variable used throughout this notebook
# This is the same as statement_policy_panel — just an alias for clarity
statements_labeled = statement_policy_panel.copy()

print(f"✓ Matched {len(statements_labeled)} FOMC statements to target rates")
print(f"  Coverage: {statements_labeled['Date'].min().date()} to {statements_labeled['Date'].max().date()}")
print(f"\nPolicy Outcome Distribution:")
print(statements_labeled['Policy_Outcome'].value_counts())

# Visualize policy decisions by year
fig, ax = plt.subplots(figsize=(14, 5))
yearly = statements_labeled.groupby(
    [statements_labeled['Date'].dt.year, 'Policy_Outcome']
).size().unstack(fill_value=0)

yearly.plot(kind='bar', stacked=True, ax=ax, color=['#C1666B', '#48A9A6', '#4357AD'])
ax.set_xlabel('Year', fontsize=11)
ax.set_ylabel('Number of Meetings', fontsize=11)
ax.set_title('FOMC Policy Decisions (2009-2026, FRED Data)', fontsize=13, fontweight='bold')
ax.legend(title='Decision')
ax.grid(axis='y', alpha=0.3)
plt.xticks(rotation=45)
plt.tight_layout()
plt.show()

url = "https://raw.githubusercontent.com/ds-modules/Small_Models_SP26/main/lance/fredgraph.png"
display(Image(url=url))

'''
Description:
This chart shows the U.S. Federal Funds Effective Rate from 2021 to early 2026. Rates remained near 0% through early 2022, rose sharply throughout 2022–2023 to above 5% as the Federal Reserve tightened monetary policy to combat inflation, and then gradually declined through 2024–2025.

Explore the data:
https://fred.stlouisfed.org/series/FEDFUNDS
'''

def chunk_text(text, chunk_size=500, overlap=25):
    """Split text into overlapping chunks."""
    words = text.split()
    return [' '.join(words[i:i + chunk_size]) for i in range(0, len(words), chunk_size - overlap)]

# Build chunked corpus using itertuples (faster than iterrows)
chunked_corpus = []
for row in df.itertuples():
    for i, chunk in enumerate(chunk_text(row.Text)):
        chunked_corpus.append({
            'text': chunk,
            'date': row.Date,
            'doc_type': row.Type,
            'chunk_id': f"{row.Date.date()}_{row.Type}_{i}"
        })

print(f"✓ Created {len(chunked_corpus):,} chunks")

from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity

print("Loading embedding model...")
embedding_model = SentenceTransformer('paraphrase-MiniLM-L3-v2')
print(f"✓ Loaded (dim={embedding_model.get_sentence_embedding_dimension()})")

print("Computing embeddings...")
corpus_texts = [chunk['text'] for chunk in chunked_corpus]
corpus_embeddings = embedding_model.encode(corpus_texts, batch_size=64, show_progress_bar=True)
print(f"✓ Embeddings: {corpus_embeddings.shape}")

class VectorIndex:
    """Vector search index with pre-normalized embeddings for fast cosine similarity."""
    def __init__(self, embeddings, metadata):
        norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
        self.embeddings = embeddings / norms  # normalize once at build time
        self.metadata = metadata

    def search(self, query_embedding, top_k=5):
        query_norm = query_embedding / np.linalg.norm(query_embedding)
        similarities = self.embeddings @ query_norm          # plain matmul, no norm recomputation
        top_k_idx = np.argpartition(similarities, -top_k)[-top_k:]  # O(n) vs O(n log n)
        top_indices = top_k_idx[np.argsort(similarities[top_k_idx])[::-1]]
        return [{'score': float(similarities[idx]), 'metadata': self.metadata[idx]} for idx in top_indices]

vector_index = VectorIndex(corpus_embeddings, chunked_corpus)
print("✓ Vector index built")

def keyword_search(query, corpus, top_k=5):
    """Simple keyword-based retrieval."""
    keywords = query.lower().split()
    scores = np.array([sum(chunk['text'].lower().count(kw) for kw in keywords) for chunk in corpus])
    top_k_idx = np.argpartition(scores, -top_k)[-top_k:]  # O(n) vs O(n log n)
    top_indices = top_k_idx[np.argsort(scores[top_k_idx])[::-1]]
    return [{'score': int(scores[idx]), 'metadata': corpus[idx]} for idx in top_indices]

print("✓ Keyword search ready")

!ls /home/jovyan/shared

from llama_cpp import Llama
import torch, os

model_path = "/home/jovyan/shared/qwen2-1_5b-instruct-q4_0.gguf"
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Device: {device}")

# Configure GPU layers — offload all layers to GPU if available, else CPU only
n_gpu_layers = -1 if device == "cuda" else 0

# Set threads for CPU inference
n_threads = os.cpu_count() or 4

print(f"Loading {model_path}...")
llm = Llama(
    model_path=model_path,
    n_ctx=4096,
    n_gpu_layers=n_gpu_layers,
    n_threads=n_threads,
    verbose=False,
)

label = "GPU (CUDA)" if device == "cuda" else "CPU"
print(f"✓ Model loaded ({label}, Q4_0 quantized)")

RAG_PROMPT = """You are analyzing FOMC communications. Answer using ONLY the excerpts below.
RULES:
1. Use ONLY information from excerpts
2. Cite: (FOMC, YYYY-MM-DD, statement/minutes)
3. If insufficient info, say: "Insufficient information"
4. No outside knowledge
EXCERPTS:
{excerpts}
QUESTION: {question}
ANSWER:"""

def format_chunks(results):
    return "\n\n".join(
        f"[{i}] (FOMC, {r['metadata']['date'].date()}, {r['metadata']['doc_type']})\n{r['metadata']['text']}"
        for i, r in enumerate(results, 1)
    )

def generate(prompt, max_tokens=100):
    """Generate response using llama-cpp. Greedy decoding for deterministic, fast inference."""
    response = llm.create_chat_completion(
        messages=[{"role": "user", "content": prompt}],
        max_tokens=max_tokens,
        temperature=0.0,  # greedy / deterministic
    )
    return response["choices"][0]["message"]["content"].strip()

def generate_with_rag(question, retrieved_chunks):
    """Generate with RAG."""
    prompt = RAG_PROMPT.format(excerpts=format_chunks(retrieved_chunks), question=question)
    return generate(prompt)

def generate_no_retrieval(question):
    """Generate without retrieval."""
    return generate(f"Answer this question about FOMC communications: {question}")

print("✓ Generation functions ready")

# Corpus-verified evaluation questions
eval_questions = [
    {
        'id': 1,
        'question': 'What target range did the Committee maintain in January 2026?',
        'expected': '3-1/2 to 3-3/4 percent',
        'pattern': r'3[\s-]*1/2.*3[\s-]*3/4',
        'verifiable': True
    },
    {
        'id': 2,
        'question': 'By how much did the Committee lower the target range in December 2025?',
        'expected': '1/4 percentage point',
        'pattern': r'1/4.*percentage point|25.*basis points',
        'verifiable': True
    },
    {
        'id': 3,
        'question': 'How did the Committee describe inflation in December 2025?',
        'expected': 'somewhat elevated',
        'pattern': r'somewhat elevated',
        'verifiable': True
    },
    {
        'id': 4,
        'question': 'Who voted against the January 2026 action?',
        'expected': 'Stephen I. Miran and Christopher J. Waller',
        'pattern': r'Miran.*Waller|Waller.*Miran',
        'verifiable': True
    },
    {
        'id': 5,
        'question': 'What balance sheet action did the Committee announce in December 2025?',
        'expected': 'initiate purchases of shorter-term Treasury securities',
        'pattern': r'purchases.*Treasury.*securities|Treasury.*purchases',
        'verifiable': True
    },
    {
        'id': 6,
        'question': 'What is the Fed\'s GDP growth forecast for 2027?',
        'expected': 'UNANSWERABLE',
        'pattern': None,
        'verifiable': False
    }
]

print(f"Evaluation set: {len(eval_questions)} questions")
print(f"  Corpus-extractable: {sum(q['verifiable'] for q in eval_questions)}")
print(f"  Refusal tests: {sum(not q['verifiable'] for q in eval_questions)}")

comparison_results = []

print("Running baseline vs RAG comparison...\n")

for q in eval_questions:
    print(f"Q{q['id']}: {q['question'][:60]}...")

    # Encode query once; shared across all retrieval methods
    query_emb = embedding_model.encode(q['question'])

    # Method 1: No retrieval
    no_ret_answer = generate_no_retrieval(q['question'])

    # Method 2: Keyword retrieval
    kw_retrieved = keyword_search(q['question'], chunked_corpus, top_k=3)
    kw_answer = generate_with_rag(q['question'], kw_retrieved)

    # Method 3: Semantic RAG
    sem_retrieved = vector_index.search(query_emb, top_k=3)
    sem_answer = generate_with_rag(q['question'], sem_retrieved)

    comparison_results.append({
        'id': q['id'],
        'question': q['question'],
        'expected': q['expected'],
        'pattern': q['pattern'],
        'verifiable': q['verifiable'],
        'no_ret_answer': no_ret_answer,
        'kw_answer': kw_answer,
        'sem_answer': sem_answer,
        'kw_score': kw_retrieved[0]['score'] if kw_retrieved else 0,
        'sem_score': sem_retrieved[0]['score']
    })

    print(f"  ✓ Generated 3 responses\n")

print("✓ All comparisons complete")

def check_citation(answer):
    """Check for proper citation format."""
    pattern = r'\(FOMC,\s*\d{4}-\d{2}-\d{2},\s*(statement|minutes|Statement|Minute)\)'
    return bool(re.search(pattern, answer))

def check_refusal(answer):
    """Check if model properly refused."""
    patterns = ['insufficient', 'cannot answer', 'not available', 'do not have', 'not in']
    return any(p in answer.lower() for p in patterns)

def grade_answer(answer, expected, pattern, verifiable):
    """Grade answer with regex pattern matching."""
    # Check citation
    has_citation = check_citation(answer)

    # Check correctness
    if not verifiable:
        # Should refuse
        correct = check_refusal(answer)
        grounded = correct
    else:
        # Should extract fact
        if pattern:
            # Use regex pattern
            correct = bool(re.search(pattern, answer, re.IGNORECASE))
        else:
            # Fallback: check if expected appears
            correct = expected.lower() in answer.lower()

        grounded = has_citation

    return {
        'correct': correct,
        'citation': has_citation,
        'grounded': grounded
    }

# Grade all answers for all methods
for r in comparison_results:
    r['no_ret_grade'] = grade_answer(r['no_ret_answer'], r['expected'], r['pattern'], r['verifiable'])
    r['kw_grade'] = grade_answer(r['kw_answer'], r['expected'], r['pattern'], r['verifiable'])
    r['sem_grade'] = grade_answer(r['sem_answer'], r['expected'], r['pattern'], r['verifiable'])

print("✓ Grading complete")

# Build comparison table
comparison_table = []

for r in comparison_results:
    comparison_table.append({
        'ID': r['id'],
        'Question': r['question'][:45] + '...',
        'No_Ret_Correct': '✓' if r['no_ret_grade']['correct'] else '✗',
        'No_Ret_Citation': '✓' if r['no_ret_grade']['citation'] else '✗',
        'KW_Correct': '✓' if r['kw_grade']['correct'] else '✗',
        'KW_Citation': '✓' if r['kw_grade']['citation'] else '✗',
        'RAG_Correct': '✓' if r['sem_grade']['correct'] else '✗',
        'RAG_Citation': '✓' if r['sem_grade']['citation'] else '✗',
        'RAG_Score': f"{r['sem_score']:.3f}"
    })

comp_df = pd.DataFrame(comparison_table)

print("BASELINE vs RAG COMPARISON")
print("=" * 120)
print(comp_df.to_string(index=False))
print("=" * 120)

def compute_metrics(col_prefix):
    n = len(comp_df)
    acc  = (comp_df[f'{col_prefix}_Correct']  == '✓').sum() / n * 100
    cite = (comp_df[f'{col_prefix}_Citation'] == '✓').sum() / n * 100
    return {'Accuracy': acc, 'Citation': cite}

metrics_summary = pd.DataFrame({
    'No Retrieval': compute_metrics('No_Ret'),
    'Keyword':      compute_metrics('KW'),
    'Semantic RAG': compute_metrics('RAG'),
}).T

print("\nMETRICS BY METHOD")
print("=" * 50)
for row_label in metrics_summary.index:
    acc_str = f"{metrics_summary.loc[row_label, 'Accuracy']:.1f}%"
    cite_str = f"{metrics_summary.loc[row_label, 'Citation']:.1f}%"
    print(f"  {row_label:<20} Accuracy: {acc_str}   Citation: {cite_str}")
print("=" * 50)

import difflib

def sentence_split(text):
    return [s.strip() for s in re.split(r'(?<=[.!?])\s+', text) if s.strip()]

# Get two most recent consecutive statements
sorted_stmts = statements_labeled.sort_values('Date').reset_index(drop=True)
current  = sorted_stmts.iloc[-1]
previous = sorted_stmts.iloc[-2]

# Use difflib to identify textual changes between statements
curr_sents = sentence_split(current['Text'])
prev_sents = sentence_split(previous['Text'])
diff    = list(difflib.Differ().compare(prev_sents, curr_sents))
added   = [line[2:] for line in diff if line.startswith('+ ')]
removed = [line[2:] for line in diff if line.startswith('- ')]

print(f"Actual changes: {len(added)} added, {len(removed)} removed")
print(f"Policy change: {previous['Policy_Outcome']} → {current['Policy_Outcome']}\n")

# Test 1: Identify changes
diff_prompt_1 = f"""Compare these FOMC statements and list the key textual changes:

PREVIOUS ({previous['Date'].date()}):
{previous['Text'][:500]}...

CURRENT ({current['Date'].date()}):
{current['Text'][:500]}...

What specific language changed?"""

diff_answer_1 = generate(diff_prompt_1, max_tokens=120)

print("DIFFERENCING TEST 1: Identify Changes")
print("-" * 80)
print(f"Model response:\n{diff_answer_1}\n")

# Test 2: Policy stance
diff_prompt_2 = f"""Did the policy stance become more hawkish or dovish from {previous['Date'].date()} to {current['Date'].date()}?

Context: Previous decision was {previous['Policy_Outcome']}, current is {current['Policy_Outcome']}."""

diff_answer_2 = generate(diff_prompt_2, max_tokens=80)

print("DIFFERENCING TEST 2: Policy Stance")
print("-" * 80)
print(f"Model response:\n{diff_answer_2}\n")

# Score differencing performance
def score_differencing(answer, added_sents, removed_sents, policy_change):
    """Score differencing answer."""
    answer_lower = answer.lower()

    # Check if identified additions
    identified_add = any(s[:30].lower() in answer_lower for s in added_sents[:3])

    # Check if identified removals
    identified_remove = any(s[:30].lower() in answer_lower for s in removed_sents[:3])

    # Check if linked to policy change
    linked_policy = policy_change.lower() in answer_lower

    return {
        'identified_additions': identified_add,
        'identified_removals': identified_remove,
        'linked_to_policy': linked_policy,
        'score': sum([identified_add, identified_remove, linked_policy])
    }

diff_score = score_differencing(diff_answer_1, added, removed, current['Policy_Outcome'])

print("DIFFERENCING PERFORMANCE SCORE")
print("=" * 50)
print(f"Identified additions: {'✓' if diff_score['identified_additions'] else '✗'}")
print(f"Identified removals: {'✓' if diff_score['identified_removals'] else '✗'}")
print(f"Linked to policy: {'✓' if diff_score['linked_to_policy'] else '✗'}")
print(f"Overall score: {diff_score['score']}/3")
print("=" * 50)

def normalize_prediction(text):
    """Map raw model output to Raise / Hold / Lower."""
    t = text.lower()
    if any(w in t for w in ['raise', 'increas']):
        return 'Raise'
    if any(w in t for w in ['lower', 'cut', 'decreas', 'reduc']):
        return 'Lower'
    if any(w in t for w in ['hold', 'maintain', 'unchanged']):
        return 'Hold'
    return 'Unknown'

# Sample 10 statements for classification
test_statements = statements_labeled.sample(n=10, random_state=42).reset_index(drop=True)

classification_results = []

print("Running policy classification...\n")

for idx, row in test_statements.iterrows():
    prompt = f"""Classify the FOMC policy decision as: Raise, Hold, or Lower.

Statement excerpt:
{row['Text'][:400]}

Decision (one word):"""

    prediction = generate(prompt, max_tokens=10)
    pred_class = normalize_prediction(prediction)
    correct    = (pred_class == row['Policy_Outcome'])

    classification_results.append({
        'date':      row['Date'].date(),
        'true':      row['Policy_Outcome'],
        'predicted': pred_class,
        'correct':   correct
    })

    if idx < 5:
        print(f"{row['Date'].date()}: {row['Policy_Outcome']} → {pred_class} {'✓' if correct else '✗'}")

print("\n✓ Classification complete")

class_df = pd.DataFrame(classification_results)

print("\nPOLICY OUTCOME CLASSIFICATION RESULTS")
print("=" * 60)
print(class_df.to_string(index=False))

accuracy = class_df['correct'].mean() * 100
print(f"\nClassification Accuracy: {accuracy:.1f}%")
print("=" * 60)

# Confusion matrix
from sklearn.metrics import confusion_matrix

valid = class_df[class_df['predicted'] != 'Unknown']

if len(valid) > 0:
    cm = confusion_matrix(valid['true'], valid['predicted'], labels=['Raise', 'Hold', 'Lower'])

    fig, ax = plt.subplots(figsize=(8, 6))
    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
                xticklabels=['Raise', 'Hold', 'Lower'],
                yticklabels=['Raise', 'Hold', 'Lower'], ax=ax)
    ax.set_xlabel('Predicted')
    ax.set_ylabel('True')
    ax.set_title('Policy Outcome Confusion Matrix')
    plt.tight_layout()
    plt.show()

# Reuse compute_metrics for all three methods
no_ret = compute_metrics('No_Ret')
kw     = compute_metrics('KW')
rag    = compute_metrics('RAG')

policy_acc = class_df['correct'].mean() * 100

print("FINAL EXPERIMENTAL RESULTS")
print("=" * 80)
model_label = "Q4_0 with CUDA offload" if device == "cuda" else "Q4_0 CPU"
print(f"Model: Qwen2.5-1.5B-Instruct ({model_label})")
print(f"Corpus: {len(df)} FOMC documents ({df['Date'].min().year}-{df['Date'].max().year})")
print(f"Ground truth: {len(statements_labeled)} labeled policy decisions (FRED 2009-2026)")
print(f"\nQA ACCURACY (6 questions):")
print(f"  No Retrieval:    {no_ret['Accuracy']:.1f}%")
print(f"  Keyword:         {kw['Accuracy']:.1f}%")
print(f"  Semantic RAG:    {rag['Accuracy']:.1f}%")
print(f"\nCITATION RATE:")
print(f"  No Retrieval:    {no_ret['Citation']:.1f}%")
print(f"  Keyword:         {kw['Citation']:.1f}%")
print(f"  Semantic RAG:    {rag['Citation']:.1f}%")
print(f"\nPOLICY CLASSIFICATION (10 statements):")
print(f"  Accuracy:        {policy_acc:.1f}%")
print(f"\nDIFFERENCING PERFORMANCE:")
print(f"  Score:           {diff_score['score']}/3")
print("=" * 80)