pattern-language-parser-generator/setup_resources.py at main · mattbriggs/pattern-language-parser-generator · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
import nltk
import logging
from sentence_transformers import SentenceTransformer


def download_nltk_resources():
    try:
        nltk.data.find("tokenizers/punkt")
        logging.info("NLTK 'punkt' tokenizer is already available.")
    except LookupError:
        logging.info("Downloading NLTK 'punkt' tokenizer...")
        nltk.download("punkt")


def test_sentence_transformer_model(model_name="all-MiniLM-L6-v2"):
    try:
        _ = SentenceTransformer(model_name)
        logging.info(f"SentenceTransformer model '{model_name}' is available.")
    except Exception as e:
        logging.error(f"Failed to load SentenceTransformer model: {e}")

def ensure_nltk_resource(resource):
    try:
        nltk.data.find(resource)
    except LookupError:
        print(f"📦 Downloading missing resource: {resource}")
        nltk.download(resource.split("/")[-1])


if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    logging.info("📦 Checking NLP resources for Pattern Language Miner...")
    download_nltk_resources()
    ensure_nltk_resource('tokenizers/punkt')
    ensure_nltk_resource('taggers/averaged_perceptron_tagger')
    test_sentence_transformer_model()