CLIP & Contrastive Learning

# Inference với CLIP — zero-shot classification ảnh ẩm thực Việt
import torch
from transformers import CLIPProcessor, CLIPModel
from PIL import Image

# 1. Tải mô hình tiền huấn luyện của OpenAI
device = "cuda" if torch.cuda.is_available() else "cpu"
model = CLIPModel.from_pretrained("openai/clip-vit-large-patch14").to(device)
processor = CLIPProcessor.from_pretrained("openai/clip-vit-large-patch14")
model.eval()

# 2. Chuẩn bị input
image = Image.open("mon-an.jpg").convert("RGB")

# Dùng prompt template — cải thiện ~5% accuracy
labels_raw = ["pho bo", "banh mi", "com tam", "bun cha"]
labels = [f"a photo of {lbl}, a Vietnamese dish" for lbl in labels_raw]

inputs = processor(
    text=labels,
    images=image,
    return_tensors="pt",
    padding=True,
).to(device)

# 3. Forward pass — lấy logits_per_image đã chia theo temperature
with torch.inference_mode():
    outputs = model(**inputs)
    logits = outputs.logits_per_image  # shape: [1, num_labels]
    probs = logits.softmax(dim=-1)

# 4. In kết quả — xếp theo xác suất giảm dần
scored = sorted(
    zip(labels_raw, probs[0].cpu().tolist()),
    key=lambda x: x[1],
    reverse=True,
)
for label, p in scored:
    print(f"{label:10s} → {p*100:5.1f}%")

# 5. Truy cập embedding trực tiếp (ví dụ cho image retrieval)
image_features = model.get_image_features(**{
    k: v for k, v in inputs.items() if k in ("pixel_values",)
})
text_features = model.get_text_features(**{
    k: v for k, v in inputs.items()
    if k in ("input_ids", "attention_mask")
})

# L2-normalize rồi cosine = dot product
image_features = image_features / image_features.norm(dim=-1, keepdim=True)
text_features = text_features / text_features.norm(dim=-1, keepdim=True)

similarity = image_features @ text_features.T  # [1, num_labels]
print("Cosine similarity:", similarity.cpu().tolist())