Test Notebooks

# Ensure that your python environment has huggingface_hub package installed.
import torch


# Check for GPU/MPS
device = (
    "cuda"
    if torch.cuda.is_available()
    else "mps"
    if torch.backends.mps.is_available()
    else "cpu"
)
print(f"Using device: {device}")

# Path for Shared Hub - change this to match your JupyterHub's shared directory
# Examples: /home/jovyan/shared, /home/jovyan/shared_readwrite, /home/jovyan/_shared/course-name
shared_model_path = "/home/jovyan/shared-readwrite"

from huggingface_hub import HfApi, list_models
# Search for GGUF models
api = HfApi()

# Find models with "gguf" in the name, sorted by downloads
models = list(api.list_models(
    search="gguf",
    sort="downloads",
    limit=20
))

print("Top 20 GGUF models by downloads:")
print("-" * 60)
for model in models:
    print(f"{model.id}")

# List files in a specific repository to find available quantizations
from huggingface_hub import list_repo_files

repo_id = "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF"
files = list_repo_files(repo_id)

print(f"Files in {repo_id}:")
print("-" * 60)
for f in files:
    if f.endswith(".gguf"):
        print(f)

from llama_cpp import Llama
import os
model_directory = "/home/jovyan/shared/"
# Define the model filename
model_name = "qwen2-1_5b-instruct-q4_0.gguf"

# Create the full path to the model
model_path = os.path.join(model_directory, model_name)
model = Llama(
    model_path=model_path,
    n_ctx=2048,          # Context window size
    n_threads=1,      # Auto-detect optimal thread count
    n_gpu_layers=-1,     # -1 means send all layers to GPU
    verbose=True,        # Print model loading info
    chat_format="chatml" # Qwen uses ChatML format
)