Frequencies tk

followthemoney_compare/lib/word_frequency.py

@@ -29,6 +29,7 @@ def preprocess_text(text):
 class WordFrequency:
     def __init__(self, confidence, error_rate, bin_dtype="uint32"):
         self.n_items = 0
+        self.n_documents = 0
         self.confidence = confidence
         self.error_rate = error_rate
 
@@ -46,6 +47,9 @@ def iter_idxs(self, key):
             idx = (k1 + i * k2) % self.width
             yield (idx % self.width) + (i * self.width)
 
+    def add_doc(self):
+        self.n_documents += 1
+
     def add(self, key):
         idxs = list(self.iter_idxs(key))
         return self.add_idxs(idxs)
@@ -131,6 +135,7 @@ def from_proxies(
         )
         for i, proxy in enumerate(proxies):
             collection = proxy.context.get("collection")
+            tf.add_doc()
             for name in proxy.names:
                 for token in preprocess_text(name):
                     idxs = list(tf.iter_idxs(token))
@@ -139,12 +144,12 @@ def from_proxies(
                     sf[proxy.schema.name].add_idxs(idxs)
             if checkpoint_freq and (i + 1) % checkpoint_freq == 0:
                 idf_merge = WordFrequency.merge(*idf.values(), binarize=True)
-                freq = cls(tf, idf_merge, sf)
+                freq = cls(tf, sf, idf_merge)
                 freq.summarize()
                 if checkpoint_dir:
                     freq.save_dir(checkpoint_dir)
         idf_merge = WordFrequency.merge(*idf.values(), binarize=True)
-        freq = cls(tf, idf_merge, sf)
+        freq = cls(tf, sf, idf_merge)
         if checkpoint_dir:
             freq.save_dir(checkpoint_dir)
         return freq
@@ -175,7 +180,8 @@ def tfidf(self, token, schema=None):
         return self.token_frequency(token, schema) / self.document_frequency(token)
 
     def inv_tfidf(self, token, schema=None):
-        return self.document_frequency(token) / self.token_frequency(token, schema)
+        return self.token.n_documents / self.document_frequency(token)
+        # return self.document_frequency(token) / self.token_frequency(token, schema)
 
     @classmethod
     def load_dir(cls, directory):

followthemoney_compare/metrics/names.py

@@ -16,7 +16,7 @@ def _compare_names_fuzzy_wf(t1, s1, t2, s2, frequencies):
     for left, schema, right in ((tp1, s1, tp2), (tp2, s2, tp1)):
         right_str = " ".join(right)
         for token_left in left:
-            n = math.log1p(frequencies.inv_tfidf(token_left, schema=schema))
+            n = math.log(frequencies.inv_tfidf(token_left, schema=schema))
             s = partial_token_set_ratio(
                 token_left, right_str, force_ascii=False, full_process=False
             )