Merge pull request #247 from zh-lx/feature-dict-key-length

feat: 自定义词典词 length 适配双字节字符
zh-lx · Jun 9, 2024 · 150c5f0 · 150c5f0
2 parents 5d2219e + a99b29f
commit 150c5f0
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 6 deletions.
diff --git a/lib/common/segmentit/index.ts b/lib/common/segmentit/index.ts
@@ -121,10 +121,10 @@ export class AC {
   }
 
   addNodeToQueues(trieNode: TrieNode) {
-    if (!this.queues[trieNode.prefix.length]) {
-      this.queues[trieNode.prefix.length] = [];
+    if (!this.queues[stringLength(trieNode.prefix)]) {
+      this.queues[stringLength(trieNode.prefix)] = [];
     }
-    this.queues[trieNode.prefix.length].push(trieNode);
+    this.queues[stringLength(trieNode.prefix)].push(trieNode);
   }
 
   // 按照优先级插入 pattern

diff --git a/lib/data/special.ts b/lib/data/special.ts
@@ -6,6 +6,7 @@ import {
   getPinyinWithoutTone,
 } from '@/core/pinyin/handle';
 import DICT1 from './dict1';
+import { stringLength } from '@/common/utils';
 
 export const InitialList = [
   'zh',
@@ -167,7 +168,7 @@ export const PatternNumberDict: Pattern[] = Object.keys(NumberDict).map(
     zh: key,
     pinyin: NumberDict[key],
     probability: Probability.Rule,
-    length: key.length,
+    length: stringLength(key),
     priority: Priority.Normal,
     dict: Symbol('rule'),
   })

diff --git a/lib/data/surname.ts b/lib/data/surname.ts
@@ -1,5 +1,7 @@
 import { Probability, Priority } from '@/common/constant';
+import { stringLength } from '@/common/utils';
 import type { Pattern } from  '../common/segmentit';
+
 const Surnames: { [key: string]: string } = {
   南宫: 'nán gōng',
   第五: 'dì wǔ',
@@ -498,8 +500,8 @@ export default Surnames;
 export const PatternSurname: Pattern[] = Object.keys(Surnames).map((key) => ({
   zh: key,
   pinyin: Surnames[key],
-  probability: Probability.Surname + key.length,
-  length: key.length,
+  probability: Probability.Surname + stringLength(key),
+  length: stringLength(key),
   priority: Priority.Surname,
   dict: Symbol('surname'),
 }));