test_word_tokenize.py 285 Bytes Edit 1 2 3 4 5 6 7 8 9 10 11 12 def word_tokenize(title: str): """ 分词器 """ from nltk.tokenize import word_tokenize result = word_tokenize(title, "english") return result if __name__ == '__main__': aba = "nation's bravest tales of courage and heroism" print(word_tokenize(aba))
