gguf-vectors.py

import numpy as np
from txtai.ann import ANNFactory

# Index 10M vectors using llama.cpp style quants
ann = None
for _ in range(1000):
  # Generate batch of vectors
  batch = np.random.rand(10000, 768).astype(np.float32)

  if not ann:
    ann = ANNFactory.create({
      "backend": "ggml",
      "ggml": {
        "quantize": "MXFP4"
      }
    })

    ann.index(batch)
  else:
    ann.append(batch)