feat: Add support for JiebaTokenSplitter in tokenizer.py

urchade · Jun 27, 2024 · d549637 · d549637
1 parent 8ea5cba
commit d549637
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/gliner/data_processing/tokenizer.py b/gliner/data_processing/tokenizer.py
@@ -86,7 +86,7 @@ def __init__(self, splitter_type='whitespace'):
         elif splitter_type == 'jieba':
             self.splitter = JiebaTokenSplitter()
         else:
-            raise ValueError(f"{splitter_type} is not implemented, choose between 'whitespace', 'spacy' and 'mecab'")
+            raise ValueError(f"{splitter_type} is not implemented, choose between 'whitespace', 'spacy', 'jieba' and 'mecab'")
 
     def __call__(self, text):
         for token in self.splitter(text):