Distinctiveness Analysis backend

backend/app/admin.py

@@ -5,14 +5,28 @@
 from django.contrib import admin
 from . import models
 
+from .models import Corpus, DistinctivenessAnalysis
+
+
+class CorpusAdmin(admin.ModelAdmin):
+    list_display = ('pk', 'title')
+
+
+class DistinctivenessAdmin(admin.ModelAdmin):
+    list_display = ('pk', 'corpus_1_id', 'corpus_2_id')
+
+
 models_to_register = [
     models.Document,
     models.PronounSeries,
     models.Gender,
-    models.Corpus,
     models.ProximityAnalysis,
     models.FrequencyAnalysis,
 ]
 
 for model in models_to_register:
     admin.site.register(model)
+
+admin.site.register(Corpus, CorpusAdmin)
+admin.site.register(DistinctivenessAnalysis, DistinctivenessAdmin)
+

backend/app/analysis/distinctiveness.py

@@ -0,0 +1,119 @@
+import math
+from collections import Counter
+
+from ..models import Corpus
+
+
+####################
+# HELPER FUNCTIONS #
+####################
+
+
+def _get_wordcount_counter(corpus: Corpus):
+    """
+    This function returns a Counter object that stores
+    how many times each word appears in the corpus.
+    :return: Python Counter object
+    """
+    corpus_counter = Counter()
+    for document in corpus.documents.all():
+        document_counter = document.word_count_counter
+        corpus_counter += document_counter
+    return corpus_counter
+
+
+def dunn_individual_word(total_words_in_corpus_1,
+                         total_words_in_corpus_2,
+                         count_of_word_in_corpus_1,
+                         count_of_word_in_corpus_2):
+    """
+    applies Dunning log likelihood to compare individual word in two counter objects
+
+    :param total_words_in_corpus_1: int, total wordcount in corpus 1
+    :param total_words_in_corpus_2: int, total wordcount in corpus 2
+    :param count_of_word_in_corpus_1: int, wordcount of one word in corpus 1
+    :param count_of_word_in_corpus_2: int, wordcount of one word in corpus 2
+    :return: Float representing the Dunning log likelihood of the given inputs
+
+    >>> total_words_m_corpus = 8648489
+    >>> total_words_f_corpus = 8700765
+    >>> wordcount_female = 1000
+    >>> wordcount_male = 50
+    >>> dunn_individual_word(total_words_m_corpus,
+    ...                      total_words_f_corpus,
+    ...                      wordcount_male,
+    ...                      wordcount_female)
+    -1047.8610274053995
+
+    """
+    # NOTE(ra): super short var names actually useful here for reading the math
+    # pylint: disable=invalid-name
+
+    a = count_of_word_in_corpus_1
+    b = count_of_word_in_corpus_2
+    c = total_words_in_corpus_1
+    d = total_words_in_corpus_2
+
+    e1 = c * (a + b) / (c + d)
+    e2 = d * (a + b) / (c + d)
+
+    dunning_log_likelihood = 2 * (a * math.log(a / e1) + b * math.log(b / e2))
+
+    if count_of_word_in_corpus_1 * math.log(count_of_word_in_corpus_1 / e1) < 0:
+        dunning_log_likelihood = -dunning_log_likelihood
+
+    return dunning_log_likelihood
+
+
+def dunning_total(corpus_1, corpus_2):
+    """
+    Runs dunning_individual on words shared by both Corpus objects
+    (-) end of spectrum is words for corpus_2
+    (+) end of spectrum is words for corpus_1
+    the larger the magnitude of the number, the more distinctive that word is in its
+    respective Corpus object
+
+    :param corpus_1: Python Corpus object
+    :param corpus_2: Python Corpus object
+    :return: Dictionary of the form
+        {"unique to corp 1": {word1, word2, word3, ...},
+        "unique to corp 2": {word1, word2, word3, ...},
+        "common words": {word1: {dunning score, count1, count2}, word2: {dunning score, count1, count 2}, ...}}
+    """
+    counter_1 = _get_wordcount_counter(corpus_1)
+    counter_2 = _get_wordcount_counter(corpus_2)
+
+    total_word_count_1 = sum(counter_1.values())
+    total_word_count_2 = sum(counter_2.values())
+
+    assert total_word_count_1 > 0 or total_word_count_2 > 0, "One or more corpora is empty"
+
+    # dictionary where results will be returned
+    result = {"unique_to_corp_1": set(),
+              "unique_to_corp_2": set(),
+              "common_words": {}}
+
+    for word in counter_1:
+        counter1_wordcount = counter_1[word]
+        try:
+            counter2_wordcount = counter_2[word]
+
+            dunning_word = dunn_individual_word(total_word_count_1,
+                                                total_word_count_2,
+                                                counter1_wordcount,
+                                                counter2_wordcount)
+
+            result["common_words"][word] = {
+                'dunning': dunning_word,
+                'count_corp1': counter1_wordcount,
+                'count_corp2': counter2_wordcount
+            }
+        except ValueError:  # if word is not in counter_2
+            result["unique_to_corp_1"].add((word, counter1_wordcount))
+
+    # consider words unique to corpus_2
+    for word in counter_2:
+        if word not in counter_1:
+            result["unique_to_corp_2"].add((word, counter_2[word]))
+
+    return result

backend/app/migrations/0012_distinctivenessanalysis.py

@@ -0,0 +1,26 @@
+# Generated by Django 3.1.5 on 2021-11-22 21:01
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('app', '0011_frequencyanalysis'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DistinctivenessAnalysis',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('results', models.JSONField()),
+                ('corpus_1', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='dist_analysis_corpora_1', to='app.corpus')),
+                ('corpus_2', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='dist_analysis_corpora_2', to='app.corpus')),
+            ],
+            options={
+                'verbose_name_plural': 'Distinctiveness Analyses',
+            },
+        ),
+    ]

backend/app/migrations/0013_auto_20211122_1635.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.1.5 on 2021-11-22 21:35
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('app', '0012_distinctivenessanalysis'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='distinctivenessanalysis',
+            name='corpus_1',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='dist_analyses_for_corpus_1', to='app.corpus'),
+        ),
+        migrations.AlterField(
+            model_name='distinctivenessanalysis',
+            name='corpus_2',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='dist_analyses_for_corpus_2', to='app.corpus'),
+        ),
+    ]

backend/app/migrations/0014_auto_20211201_1636.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.5 on 2021-12-01 21:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('app', '0013_auto_20211122_1635'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='frequencyanalysis',
+            name='results',
+            field=models.JSONField(null=True),
+        ),
+    ]

backend/app/migrations/0015_auto_20211201_1638.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.5 on 2021-12-01 21:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('app', '0014_auto_20211201_1636'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='distinctivenessanalysis',
+            name='results',
+            field=models.JSONField(null=True),
+        ),
+    ]

backend/app/models.py

@@ -571,14 +571,27 @@ class ProximityAnalysis(models.Model):
     class Meta:
         verbose_name_plural = 'proximity analyses'
 
+
 class FrequencyAnalysis(models.Model):
     """
     This model will persist the results from the frequency analysis functions.
     """
 
     corpus = models.ForeignKey(Corpus, related_name='frequency_analyses', on_delete=models.CASCADE)
     genders = models.ManyToManyField(Gender, related_name='frequency_analyses')
-    results = models.JSONField()
+    results = models.JSONField(null=True)
 
     class Meta:
         verbose_name_plural = 'Frequency Analyses'
+
+
+class DistinctivenessAnalysis(models.Model):
+    """
+    This model will persist the results from the distinctiveness analysis functions.
+    """
+    corpus_1 = models.ForeignKey(Corpus, related_name='dist_analyses_for_corpus_1', on_delete=models.CASCADE)
+    corpus_2 = models.ForeignKey(Corpus, related_name='dist_analyses_for_corpus_2', on_delete=models.CASCADE)
+    results = models.JSONField(null=True)
+
+    class Meta:
+        verbose_name_plural = 'Distinctiveness Analyses'

backend/app/serializers.py

@@ -11,7 +11,9 @@
     Document,
     Corpus,
     ProximityAnalysis,
-    FrequencyAnalysis
+    FrequencyAnalysis,
+    DistinctivenessAnalysis
+
 )
 
 
@@ -90,3 +92,14 @@ class FrequencyAnalysisSerializer(serializers.ModelSerializer):
     class Meta:
         model = FrequencyAnalysis
         fields = ['id', 'corpus', 'genders', 'results']
+
+
+class DistinctivenessAnalysisSerializer(serializers.ModelSerializer):
+    """
+    Serializes a DistinctivenessAnalysis object
+    """
+
+    class Meta:
+        model = DistinctivenessAnalysis
+        fields = ['id', 'corpus_1', 'corpus_2', 'results']
+