rotor.py

import collections
import itertools 
import dat
import scipy

def words(file):
    with open(file) as words_file:
        while True:
            line = words_file.readline()
            if not line: break
            if len(line) <= 3: continue
            yield line.strip()

def distances(word):
    if len(word) <= 1: return ()
    distance = ()
    chars = list(word)
    current = ord(chars[0])
    for c in chars[1:]:
        oc = ord(c)
        d = oc - current if oc >= current else (26 - (current - oc))
        distance = distance + (d,)
        current = oc
    return distance

def collect(file):
    output = collections.defaultdict(list)
    for word in words(file): output[distances(word)].append(word)
    return output

def rotors(od):
    for k in od: 
        if len(od[k]) > 1: 
            yield od[k]

def cosine(model, rs):
    for r in rs:
        for p in itertools.combinations(r, 2):
            v0 = model.vectors.get(p[0])
            v1 = model.vectors.get(p[1])
            if v0 is not None and v1 is not None:
                yield scipy.spatial.distance.cosine(v0, v1), p[0], p[1]

def main():
    # https://github.com/jayolson/divergent-association-task/blob/main/words.txt
    file = "words.txt"

    print("Loading model...")
    # https://nlp.stanford.edu/data/glove.840B.300d.zip
    model = dat.Model("glove.840B.300d.txt", file)
    print("Model loaded")
          
    od = collections.OrderedDict(collect(file).items())
    rs = list(rotors(od))
    print("{} total entries with {} rotor words".format(len(od), len(rs)))
    for d, w0, w1 in sorted(list(cosine(model, rs))):
        print("{},{},{}".format(d.item(), w0, w1))

if __name__ == "__main__":
    main()

semantid-distances.csv