Skip to content

P
pdf-miner
Commit 57380cbe authored by myhloli's avatar myhloli
Browse files
Options

feat(language): add FT LANG cache directory setup

parent a396b8f0
Hide whitespace changes
Inline Side-by-side
Showing with 12 additions and 0 deletions
magic_pdf/libs/language.py
View file @ 57380cbe
import os
import unicodedata import unicodedata
if not os.getenv("FTLANG_CACHE"):
current_file_path = os.path.abspath(__file__)
current_dir = os.path.dirname(current_file_path)
root_dir = os.path.dirname(current_dir)
ftlang_cache_dir = os.path.join(root_dir, 'resources', 'fasttext-langdetect')
os.environ["FTLANG_CACHE"] = str(ftlang_cache_dir)
# print(os.getenv("FTLANG_CACHE"))
from fast_langdetect import detect_language from fast_langdetect import detect_language
def detect_lang(text: str) -> str: def detect_lang(text: str) -> str:
if len(text) == 0: if len(text) == 0:
return "" return ""
try: try:
...@@ -18,6 +29,7 @@ def detect_lang(text: str) -> str: ...@@ -18,6 +29,7 @@ def detect_lang(text: str) -> str:
if __name__ == '__main__': if __name__ == '__main__':
print(os.getenv("FTLANG_CACHE"))
print(detect_lang("This is a test.")) print(detect_lang("This is a test."))
print(detect_lang("<html>This is a test</html>")) print(detect_lang("<html>This is a test</html>"))
print(detect_lang("这个是中文测试。")) print(detect_lang("这个是中文测试。"))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment