[1] Sequential Approach

The sequential approach to topic modeling involves executing tasks one after the other, without parallel processing. In the context of Latent Dirichlet Allocation (LDA), this means training the model and calculating coherence scores in a linear fashion.

Advantages of the Sequential Approach

1. Simplicity:

   • The sequential approach is easier to understand and implement. There are fewer complexities involved, making the code more maintainable and easier to debug.

2. Deterministic Behavior:

   • Since tasks are executed in a specific order, the results are predictable and reproducible, which is important for debugging and validation.

3. Lower Overhead:

   • There is no overhead associated with managing multiple threads or processes, which can be advantageous for smaller datasets where the time saved by parallelization may not justify the complexity.

4. Resource Efficiency for Small Datasets:

   • For smaller datasets, the sequential approach can be more resource-efficient since it avoids the overhead of parallel processing.

5. Easier Error Handling:

   • With a linear execution flow, it’s often easier to handle errors and exceptions, as the context is clear and the order of operations is straightforward.

Limitations of the Sequential Approach

1. Longer Processing Time:

   • The most significant drawback is that sequential processing can be significantly slower, especially for large datasets. Each step must complete before the next begins, which can lead to longer total runtimes.

2. Inefficient Resource Utilization:

   • The sequential approach may not fully utilize available CPU resources, especially on multi-core systems, leading to underperformance.

3. Limited Scalability:

   • As the dataset size increases, the sequential approach becomes less viable due to extended processing times. This can hinder scalability and responsiveness in real-time applications.

4. Lack of Flexibility:

   • In scenarios where tasks could be performed concurrently, the sequential approach misses the opportunity for optimization and faster execution.

5. Bottlenecks:

   • If one part of the process is slow (e.g., training the LDA model), it can create a bottleneck that affects the overall performance of the pipeline.

import subprocess
import sys
import os
import matplotlib.pyplot as plt
import pickle
import datetime
from tqdm import tqdm
from gensim.models import CoherenceModel, LdaMulticore, TfidfModel
from gensim.corpora import Dictionary

# List of required packages
REQUIRED_PACKAGES = ["gensim", "matplotlib", "tqdm"]

def install_packages():
    """Install missing required packages."""
    for package in REQUIRED_PACKAGES:
        try:
            __import__(package)
        except ImportError:
            print(f"Package '{package}' is missing. Attempting to install...")
            try:
                subprocess.check_call([sys.executable, "-m", "pip", "install", package])
            except Exception as e:
                print(f"Failed to install package '{package}': {e}")

# Ensure required packages are installed
install_packages()

def train_lda_and_calculate_coherence(num_topics, corpus, dictionary, ngram_texts):
    """
    Train an LDA model and calculate coherence for a given number of topics.
    This function must be self-contained and pickleable.
    """
    lda_model = LdaMulticore(
        corpus=corpus,
        num_topics=num_topics,
        id2word=dictionary,
        random_state=42,
        passes=5,
        workers=4
    )
    coherence_model = CoherenceModel(
        model=lda_model,
        texts=ngram_texts,
        dictionary=dictionary,
        coherence='c_v'
    )
    return coherence_model.get_coherence()

def explore_topic_modeling_sequential(
    ngram_texts, path_dset, dset_name, timestamp, min_topics=2, max_topics=10, step=1, no_below=3, no_above=0.8
):
    """
    Perform coherence test and create LDA topic modeling, saving outputs with a timestamp.
    """
    # Generate timestamp and output directory
    output_dir = os.path.join(path_dset)
    os.makedirs(output_dir, exist_ok=True)

    # Create dictionary and filter extremes
    dictionary = Dictionary(ngram_texts)
    dictionary.filter_extremes(no_below=no_below, no_above=no_above)
    corpus = [dictionary.doc2bow(text) for text in ngram_texts]

    # Apply TF-IDF transformation
    tfidf = TfidfModel(corpus)
    corpus_tfidf = tfidf[corpus]

    # Save dictionary and corpus
    dictionary_path = os.path.join(output_dir, f"{dset_name}_dictionary_{timestamp}.dict")
    corpus_path = os.path.join(output_dir, f"{dset_name}_corpus_{timestamp}.pkl")
    dictionary.save(dictionary_path)
    with open(corpus_path, "wb") as f:
        pickle.dump(corpus_tfidf, f)  # Save the TF-IDF-transformed corpus

    # Define topic range
    topic_range = range(min_topics, max_topics + 1, step)

    # Check if coherence values are cached
    coherence_cache_path = os.path.join(output_dir, f"{dset_name}_coherence_cache.pkl")
    if os.path.exists(coherence_cache_path):
        with open(coherence_cache_path, "rb") as f:
            coherence_values = pickle.load(f)
    else:
        # Sequentially calculate coherence values
        coherence_values = []
        for num_topics in tqdm(topic_range, desc="LDA Topic Modeling (Sequential)"):
            coherence_value = train_lda_and_calculate_coherence(num_topics, corpus_tfidf, dictionary, ngram_texts)
            coherence_values.append(coherence_value)

        # Cache the coherence values
        with open(coherence_cache_path, "wb") as f:
            pickle.dump(coherence_values, f)

    # Find the optimal number of topics
    optimal_topic_no = topic_range[coherence_values.index(max(coherence_values))]

    # Save coherence scores
    coherence_path = os.path.join(output_dir, f"{dset_name}_coherence_scores_{timestamp}.pkl")
    with open(coherence_path, "wb") as f:
        pickle.dump(coherence_values, f)

    # Plot coherence scores
    plt.figure(figsize=(10, 5))
    plt.plot(topic_range, coherence_values, marker="o", label="Coherence Score")
    plt.xlabel("Number of Topics")
    plt.ylabel("Coherence Score")
    plt.title("Coherence Score vs. Number of Topics")
    plt.legend()
    plt.grid()

    # Save the coherence plot
    coherence_plot_path = os.path.join(output_dir, f"{dset_name}_coherence_plot_{timestamp}.png")
    plt.savefig(coherence_plot_path)
    plt.close()

    # Train the final LDA model with the optimal number of topics
    lda_model = LdaMulticore(
        corpus=corpus_tfidf,  # Use the TF-IDF-transformed corpus
        num_topics=optimal_topic_no,
        id2word=dictionary,
        random_state=42,
        passes=10,
        workers=4
    )
    lda_model_path = os.path.join(output_dir, f"{dset_name}_lda_model_{timestamp}.model")
    lda_model.save(lda_model_path)

    return {
        "dictionary": dictionary_path,
        "corpus": corpus_path,
        "lda_model": lda_model_path,
        "coherence_scores": coherence_path,
        "coherence_plot": coherence_plot_path,
        "optimal_topics": optimal_topic_no
    }

Calling sequential approach:

import datetime
ymdhm=datetime.datetime.now().strftime("%Y%m%dT%H%M")

output_paths = explore_topic_modeling_sequential(
    df_dset["Clean1NtermToken"].tolist(),
    f"{path_dset}output/TopicModeling/{ymdhm}",
    dset_name,
    ymdhm,
    no_below=3,
    no_above=0.8)

# Print saved file paths
for key, value in output_paths.items():
    print(f"{key}: {value}")

[2] Parallel Approach

The parallel approach in topic modeling, particularly when using Latent Dirichlet Allocation (LDA), involves distributing the computational workload across multiple processors or threads. This can significantly speed up the training and coherence evaluation of LDA models.

Advantages of the Parallel Approach

1. Increased Efficiency:

   • By leveraging multiple CPU cores, the time taken to train the LDA model and calculate coherence scores is reduced. This is especially beneficial when dealing with large datasets.

2. Scalability:

   • The parallel approach can handle larger datasets more effectively, making it suitable for big data applications.

3. Improved Resource Utilization:

   • It maximizes the use of available hardware resources, leading to better performance in computational tasks.

4. Faster Experimentation:

   • Researchers and data scientists can experiment with various numbers of topics in a shorter timeframe, facilitating quicker iterations and refinements in their models.

5. Error Handling:

   • As seen in the provided code, if parallel processing fails, the implementation falls back to sequential processing, ensuring that results can still be obtained even in adverse conditions.

Limitations of the Parallel Approach

1. Overhead of Parallelization:

   • There is inherent overhead in managing multiple threads or processes, which can negate the performance benefits for smaller datasets.

2. Complexity:

   • Implementing parallel processing adds complexity to the codebase, making it harder to debug and maintain.

3. Resource Contention:

   • Multiple processes can compete for limited CPU and memory resources, potentially leading to reduced performance if not managed properly.

4. Dependency on Environment:

   • The effectiveness of parallel processing can depend on the underlying hardware and software environment, which may not always be optimized for such operations.

5. Diminishing Returns:

   • Beyond a certain point, adding more workers may lead to diminishing returns in performance gain, particularly if the workload cannot be evenly distributed.

import subprocess
import sys
import os
import matplotlib.pyplot as plt
import pickle
import datetime
from tqdm import tqdm
from gensim.models import CoherenceModel, LdaMulticore
from gensim.corpora import Dictionary
from joblib import Parallel, delayed
import joblib

# List of required packages
REQUIRED_PACKAGES = ["gensim", "matplotlib", "tqdm", "joblib"]

def install_packages():
    """Install missing required packages."""
    for package in REQUIRED_PACKAGES:
        try:
            __import__(package)
        except ImportError:
            print(f"Package '{package}' is missing. Attempting to install...")
            try:
                subprocess.check_call([sys.executable, "-m", "pip", "install", package])
            except Exception as e:
                print(f"Failed to install package '{package}': {e}")

# Ensure required packages are installed
install_packages()

def train_lda_and_calculate_coherence(num_topics, corpus, dictionary, ngram_texts):
    """
    Train an LDA model and calculate coherence for a given number of topics.
    This function must be self-contained and pickleable.
    """
    lda_model = LdaMulticore(
        corpus=corpus,
        num_topics=num_topics,
        id2word=dictionary,
        random_state=42,
        passes=5,
        workers=4
    )
    coherence_model = CoherenceModel(
        model=lda_model,
        texts=ngram_texts,
        dictionary=dictionary,
        coherence='c_v'
    )
    return coherence_model.get_coherence()

def explore_topic_modeling_parallel(
    ngram_texts, path_dset, dset_name,timestamp, min_topics=2, max_topics=10, step=1, no_below=3, no_above=0.8
):
    """
    Perform coherence test and create LDA topic modeling, saving outputs with a timestamp.
    """
    # Generate timestamp and output directory
    # timestamp = datetime.datetime.now().strftime("%Y%m%dT%H%M")
    output_dir = os.path.join(path_dset)
    os.makedirs(output_dir, exist_ok=True)

    # Create dictionary and filter extremes
    dictionary = Dictionary(ngram_texts)
    dictionary.filter_extremes(no_below=no_below, no_above=no_above)
    corpus = [dictionary.doc2bow(text) for text in ngram_texts]

    # Save dictionary and corpus
    dictionary_path = os.path.join(output_dir, f"{dset_name}_dictionary_{timestamp}.dict")
    corpus_path = os.path.join(output_dir, f"{dset_name}_corpus_{timestamp}.pkl")
    dictionary.save(dictionary_path)
    with open(corpus_path, "wb") as f:
        pickle.dump(corpus, f)

    # Define topic range
    topic_range = range(min_topics, max_topics + 1, step)

    # Check if coherence values are cached
    coherence_cache_path = os.path.join(output_dir, f"{dset_name}_coherence_cache.pkl")
    if os.path.exists(coherence_cache_path):
        with open(coherence_cache_path, "rb") as f:
            coherence_values = joblib.load(f)
    else:
        # Parallelize the coherence calculation
        try:
            coherence_values = Parallel(n_jobs=-1)(
                delayed(train_lda_and_calculate_coherence)(num_topics, corpus, dictionary, ngram_texts)
                for num_topics in tqdm(topic_range, desc="LDA Topic Modeling (Parallel)")
            )
        except Exception as e:
            print(f"Error during parallel processing: {e}")
            print("Falling back to sequential processing...")
            coherence_values = [
                train_lda_and_calculate_coherence(num_topics, corpus, dictionary, ngram_texts)
                for num_topics in tqdm(topic_range, desc="LDA Topic Modeling (Sequential)")
            ]
        # Cache the coherence values
        with open(coherence_cache_path, "wb") as f:
            joblib.dump(coherence_values, f)

    # Find the optimal number of topics
    optimal_topic_no = topic_range[coherence_values.index(max(coherence_values))]

    # Save coherence scores
    coherence_path = os.path.join(output_dir, f"{dset_name}_coherence_scores_{timestamp}.pkl")
    with open(coherence_path, "wb") as f:
        pickle.dump(coherence_values, f)

    # Plot coherence scores
    plt.figure(figsize=(10, 5))
    plt.plot(topic_range, coherence_values, marker="o", label="Coherence Score")
    plt.xlabel("Number of Topics")
    plt.ylabel("Coherence Score")
    plt.title("Coherence Score vs. Number of Topics")
    plt.legend()
    plt.grid()

    # Save the coherence plot
    coherence_plot_path = os.path.join(output_dir, f"{dset_name}_coherence_plot_{timestamp}.png")
    plt.savefig(coherence_plot_path)
    plt.close()

    # Train the final LDA model with the optimal number of topics
    lda_model = LdaMulticore(
        corpus=corpus,
        num_topics=optimal_topic_no,
        id2word=dictionary,
        random_state=42,
        passes=10,
        workers=4
    )
    lda_model_path = os.path.join(output_dir, f"{dset_name}_lda_model_{timestamp}.model")
    lda_model.save(lda_model_path)

    return {
        "dictionary": dictionary_path,
        "corpus": corpus_path,
        "lda_model": lda_model_path,
        "coherence_scores": coherence_path,
        "coherence_plot": coherence_plot_path,
        "optimal_topics": optimal_topic_no
    }

Calling parallel approach:

import datetime
ymdhm=datetime.datetime.now().strftime("%Y%m%dT%H%M")

output_paths = explore_topic_modeling_parallel(
    df_dset["Clean1NtermToken"].tolist(), 
    f"{path_dset}output/TopicModeling/{ymdhm}",
    dset_name, 
    ymdhm,
    no_below=3, 
    no_above=0.8)

# Print saved file paths
for key, value in output_paths.items():
    print(f"{key}: {value}")

[3] Retrieving saved topic modeling data

Load data:

import pickle
from gensim.models import LdaModel
from gensim.corpora import Dictionary

timestamp = ymdhm 

# Load LDA Model
lda_model_path = f"{path_dset}output/TopicModeling/{timestamp}/{dset_name}_lda_model_{timestamp}.model"
lda_model = LdaModel.load(lda_model_path)

# Load Dictionary
dictionary_path = f"{path_dset}output/TopicModeling/{timestamp}/{dset_name}_dictionary_{timestamp}.dict"
dictionary = Dictionary.load(dictionary_path)

# Load Corpus
# corpus_path = f"{path_dset}output/TopicModeling/{timestamp}/{dset_name}_corpus_{timestamp}.pkl"
with open(corpus_path, "rb") as f:
    corpus = pickle.load(f)

# Load Coherence Scores
coherence_scores_path = f"{path_dset}output/TopicModeling/{timestamp}/{dset_name}_coherence_scores_{timestamp}.pkl"
with open(coherence_scores_path, "rb") as f:
    coherence_scores = pickle.load(f)

print("LDA model, dictionary, corpus, and coherence scores loaded successfully!")

Get top terms per topic

import numpy as np

def get_top_terms_per_topic(lda_model, top_n=10):
    """
    Extract top_n terms from each topic of an LDA model.

    Args:
        lda_model: Trained LDA model (e.g., from gensim.models.LdaModel).
        top_n (int): How many top terms to extract per topic.

    Returns:
        dict: A dictionary of { topic_id: [list_of_top_terms] }.
    """

    # Dictionary to store top terms by topic
    top_terms_dict = {}

    # Get the number of topics from the LDA model
    num_topics = lda_model.num_topics

    # Iterate over each topic
    for topic_id in range(num_topics):
        # Get the top words and their probabilities for this topic
        topic_terms = lda_model.show_topic(topic_id, top_n)

        # Extract only the term names (since show_topic() already returns words)
        top_terms = [term for term, _ in topic_terms]

        # Store in dictionary
        top_terms_dict[topic_id] = top_terms

    return top_terms_dict

Show topic-based terms

top_terms_dict = get_top_terms_per_topic(lda_model, top_n=10)
# Print top terms per topic
for topic_id, terms in top_terms_dict.items():
    print(f"Topic {topic_id}: {', '.join(terms)}")