浏览代码

version 1 of similarity

chenhaiyang 5 年之前
父节点
当前提交
01a456b18a
共有 5 个文件被更改,包括 100221 次插入0 次删除
  1. 0 0
      sim/__init__.py
  2. 17884 0
      sim/cilin_dict.txt
  3. 82193 0
      sim/cilin_words.txt
  4. 80 0
      sim/text_sim.py
  5. 64 0
      sim/words_sim.py

+ 0 - 0
sim/__init__.py


文件差异内容过多而无法显示
+ 17884 - 0
sim/cilin_dict.txt


文件差异内容过多而无法显示
+ 82193 - 0
sim/cilin_words.txt


+ 80 - 0
sim/text_sim.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+
+from operator import itemgetter
+
+import jieba
+import numpy as np
+from words_sim import SimCilin
+
+ci_lin = SimCilin()
+jieba.load_userdict('cilin_words.txt')
+
+
+def segmentation(sentence_list):
+    result = list()
+    for s in sentence_list:
+        temp_seg = jieba.cut(s)
+        result.append([x for x in temp_seg])
+
+    return result
+
+
+def get_similarity(s1, s2):
+    all_sim_1 = list()
+    for w1 in s1:
+        if is_contains_chinese(w1):
+            sim_list = list()
+            for w2 in s2:
+                sim_list.append(ci_lin.compute_word_sim(w1, w2))
+            sim_list.sort()
+            all_sim_1.append(sim_list[-1])
+
+    all_sim_2 = list()
+    for w1 in s2:
+        if is_contains_chinese(w1):
+            sim_list = list()
+            for w2 in s1:
+                sim_list.append(ci_lin.compute_word_sim(w1, w2))
+            sim_list.sort()
+            all_sim_2.append(sim_list[-1])
+
+    return (np.mean(all_sim_1) + np.mean(all_sim_2)) / 2
+
+
+def most_similar_items(src_s, sentences, n=3):
+    sentences = segmentation(sentences)
+    temp = list()
+    for item in sentences:
+        sim_value = get_similarity(src_s, item)
+        temp.append({
+            'key': merge(item),
+            'value': sim_value,
+        })
+    result = sorted(temp, key=itemgetter('value'), reverse=True)
+    return result[:n]
+
+
+def is_contains_chinese(s):
+    for _char in s:
+        if '\u4e00' <= _char <= '\u9fa5':
+            return True
+    return False
+
+
+def merge(word_list):
+    s = ''
+    for w in word_list:
+        s += w.split('/')[0]
+    return s
+
+
+if __name__ == '__main__':
+    str1 = '我喜欢吃苹果'
+    str2 = '他喜欢肯红薯'
+    str1_seg = jieba.cut(str1)
+    str2_seg = jieba.cut(str2)
+    str1_new = [x for x in str1_seg]
+    str2_new = [x for x in str2_seg]
+    str_l = ['我喜欢吃梨', '你喜欢吃苹果', '他喜欢吃橙子']
+    print(get_similarity(str1_new, str2_new))
+    print(most_similar_items(str1, str_l, 5))

+ 64 - 0
sim/words_sim.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+
+import codecs
+import time
+
+
+class SimCilin(object):
+
+    def __init__(self):
+        self.cilin_path = 'cilin_dict.txt'
+        self.sem_dict = self.load_semantic()
+
+    def load_semantic(self):
+        sem_dict = dict()
+        for line in codecs.open(self.cilin_path):
+            line = line.strip().split(' ')
+            sem_type = line[0]
+            words = line[1:]
+            for word in words:
+                if word not in sem_dict:
+                    sem_dict[word] = sem_type
+                else:
+                    sem_dict[word] += ';' + sem_type
+
+        for word, sem_type in sem_dict.items():
+            sem_dict[word] = sem_type.split(';')
+        return sem_dict
+
+    def compute_word_sim(self, word1, word2):
+        sems_word1 = self.sem_dict.get(word1, [])
+        sems_word2 = self.sem_dict.get(word2, [])
+        score_list = [self.compute_sem(sem_word1, sem_word2) for sem_word1 in sems_word1 for sem_word2 in sems_word2]
+        if score_list:
+            return max(score_list)
+        else:
+            return 0
+
+    @staticmethod
+    def compute_sem(sem1, sem2):
+        sem1 = [sem1[0], sem1[1], sem1[2:4], sem1[4], sem1[5:7], sem1[-1]]
+        sem2 = [sem2[0], sem2[1], sem2[2:4], sem2[4], sem2[5:7], sem2[-1]]
+        score = 0
+        for index in range(len(sem1)):
+            if sem1[index] == sem2[index]:
+                if index in [0, 1]:
+                    score += 3
+                elif index == 2:
+                    score += 2
+                elif index in [3, 4]:
+                    score += 1
+        return score / 10
+
+
+if __name__ == '__main__':
+    w1 = '歌手'
+    w2 = '演员'
+    ci_lin = SimCilin()
+    start = time.perf_counter()
+    v = 0.0
+    for i in range(20000):
+        v = ci_lin.compute_word_sim(w1, w2)
+    end = time.perf_counter()
+    print(end - start)
+    print(v)