Check preprocessing and loading to HF hub

check_preprocessing_and_load_to_hub.py

import torch
from PIL import Image
import open_clip
from transformers import CLIPImageProcessor, CLIPTokenizer, CLIPConfig, CLIPModel


config = CLIPConfig.from_pretrained("/Users/georgebredis/Downloads/MetaCLIP_b32_400m")
model = CLIPModel(config).eval()

_, _, preprocess = open_clip.create_model_and_transforms(
    "ViT-B-32-quickgelu",
    pretrained="/Users/georgebredis/Downloads/MetaCLIP_b32_400m/metaclip_b32_400m.bin",
)

image = Image.open("/Users/georgebredis/Downloads/i.png")
processed_image = preprocess(image).unsqueeze(0)

text = ["a diagram", "a dog", "a cat"]
processed_text = open_clip.tokenize(text)
text_embed_zeros = model.get_text_features(processed_text)

non_empty_mask = processed_text.abs().sum(dim=0).bool()
processed_text = processed_text[:, non_empty_mask]
# print(processed_text)

# verify pixel_values
image_processor = CLIPImageProcessor()
pixel_values = image_processor(image, return_tensors="pt").pixel_values
assert torch.allclose(pixel_values, processed_image)

# verify input_ids
tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-base-patch32")
input_ids = tokenizer(
    text, return_tensors="pt"  # , padding="max_length", max_length=77
).input_ids
#print(input_ids)
assert torch.allclose(input_ids, processed_text)

text_embed_fixed = model.get_text_features(input_ids)
assert torch.allclose(text_embed_zeros, text_embed_fixed, atol=1e-4)


from transformers import CLIPProcessor

repos = [
    "GeorgeBredis/metaclip_b32_400m",
    "GeorgeBredis/metaclip_b16_fullcc2.5b",
    "GeorgeBredis/metaclip_b32_fullcc2.5b",
    "GeorgeBredis/metaclip-h14-fullcc2.5b",
    "GeorgeBredis/metaclip-l14-fullcc2.5b",
    "GeorgeBredis/metaclip-l4-400m",
    "GeorgeBredis/metaclip_b16_400m",
]

processor = CLIPProcessor(image_processor=image_processor, tokenizer=tokenizer)

for repo in repos:
    processor.push_to_hub(repo)
# print(model)