neural network

# -*- coding: utf-8 -*-
"""rusu_najnajnovije.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1vERMkPRKQo1w0wMpLTSMbkMg34bukgJQ
"""

from google.colab import drive

# Commented out IPython magic to ensure Python compatibility.
# %%capture
# !pip install patool

import patoolib

patoolib.extract_archive('/content/drive/MyDrive/extracted_words.zip')

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import nltk

from scipy import signal
from scipy.io import wavfile
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"

import tensorflow as tf
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report

import os

X = []
y = []

# Extracting data
for dirname, _, filenames in os.walk('/content/extracted_words'):
    for filename in filenames:
        if dirname.split('/')[-1]:
            sample_rate, samples = wavfile.read(os.path.join(dirname, filename))
            frequencies, times, spectrogram = signal.spectrogram(samples, sample_rate)
            X.append(spectrogram)
            y.append(dirname.split('/')[-1])
X = np.array(X)
X = X.reshape(X.shape + (1,))

# Obtaining labels, plotting words
all_labels = nltk.FreqDist(y)
all_labels_df = pd.DataFrame({'Label': list(all_labels.keys()), 'Count': list(all_labels.values())})
num_labels = len(all_labels)
print('num labels ',num_labels)

g = all_labels_df.nlargest(columns="Count", n = 40)
plt.figure(figsize=(5,7))
ax = sns.barplot(data=g, x= "Count", y = "Label")

plt.show()

# Obtaining classes
from sklearn.preprocessing import MultiLabelBinarizer
mlb = MultiLabelBinarizer()

mlb.fit(pd.Series(y).fillna("missing").str.split(', '))
y_mlb = mlb.transform(pd.Series(y).fillna("missing").str.split(', '))
mlb.classes_

X.shape
y_mlb.shape

# Dividing data into train, validation and test sets
X_train, X_valtest, y_train, y_valtest = train_test_split(X,y_mlb,test_size=0.2, random_state=37)
X_val, X_test, y_val, y_test = train_test_split(X_valtest,y_valtest,test_size=0.5, random_state=37)
X_train.shape, X_val.shape,X_test.shape, y_train.shape, y_val.shape,y_test.shape

import keras
from keras import Sequential
from keras.layers import Dense, Conv2D, MaxPooling2D,Flatten,Dropout,BatchNormalization

# Building a model
droprate = 0.25

input_shape = (X_train.shape[1],X_train.shape[2],1)
model = Sequential()

model.add(Conv2D(512,kernel_size=(3,3),activation='relu',input_shape=input_shape,padding="same"))
model.add(BatchNormalization())
model.add(MaxPooling2D())

model.add(Conv2D(256,kernel_size=(3,3),activation='relu',padding="same"))
model.add(BatchNormalization())
model.add(MaxPooling2D())
model.add(Dropout(droprate))

model.add(Conv2D(128,kernel_size=(3,3),activation='relu',padding="same"))
model.add(BatchNormalization())
model.add(MaxPooling2D())

model.add(Flatten())
model.add(Dense(512,activation='relu'))
model.add(BatchNormalization())
model.add(Dense(256,activation='relu'))
model.add(Dropout(droprate))
model.add(Dense(10, activation='softmax'))

# Defining training parameters
(None,1,X_train.shape[1],X_train.shape[2])
model.compile(loss=keras.losses.categorical_crossentropy,
              optimizer=keras.optimizers.Adadelta( learning_rate=0.01, weight_decay=0.00001),
              metrics=['accuracy'])
model.build()
model.summary()

from keras.callbacks import EarlyStopping
epochs = 20
batch_size = 32
callbacks = [
    EarlyStopping(
        monitor='val_accuracy',
        patience=4,
        mode='max',
        verbose=1)
]

history = model.fit(X_train, y_train,
          batch_size=batch_size,
          epochs=epochs,
          verbose=1,
          validation_data=(X_val, y_val),shuffle=True,callbacks=callbacks)

# Plot accuracy and loss graphs
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

# Calculate accuracy
from sklearn.metrics import accuracy_score
model.evaluate(X_test, y_test, verbose=1)
y_pred = model.predict(X_test, batch_size=32, verbose=1)

y_pred = (y_pred == y_pred.max(axis=1)[:,None]).astype(int)

print('Model accuracy: ', accuracy_score(y_test, y_pred))

# Metrics
report_df = pd.DataFrame(classification_report(y_test, y_pred, output_dict=True)).transpose()
report_df["label"] = list(mlb.classes_) + ["micro avg","macro avg","weighted avg","samples avg"]
report_df.sort_values(by=['f1-score','support'], ascending=False)

num_classes = 10
confusion_matrix = np.zeros((num_classes, num_classes), dtype=int)

# Calculate confusion matrix
for true, pred in zip(y_test, y_pred):
    for i in range(num_classes):
        if true[i] == 1:
            for j in range(num_classes):
                if pred[j] == 1:
                    confusion_matrix[i, j] += 1

labels = ['down', 'go', 'left', 'no', 'off', 'on', 'right', 'stop', 'up',
       'yes']
a = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]

# Plot confusion matrix
plt.figure(figsize=(8, 6))
plt.imshow(confusion_matrix, cmap='Blues', interpolation='nearest')
plt.title('Confusion Matrix')
plt.colorbar()
plt.xlabel('Predicted Label')
plt.ylabel('True Label')
plt.xticks(a, labels, rotation ='vertical')
plt.yticks(a, labels, rotation ='horizontal')
plt.tight_layout()

for i in range(num_classes):
    for j in range(num_classes):
        plt.text(j, i, str(confusion_matrix[i, j]), ha='center', va='center', color='black')

plt.show()

# Saving the model on Google Drive
model.save('/content/drive/MyDrive/new_model.keras')

# Extracting data with background noise
patoolib.extract_archive('/content/drive/MyDrive/noise.zip')

A = []
b = []

for dirname, _, filenames in os.walk('/content/noise'):
    for filename in filenames:
        if dirname.split('/')[-1]:
            sample_rate, samples = wavfile.read(os.path.join(dirname, filename))
            frequencies, times, spectrogram = signal.spectrogram(samples, sample_rate)
            A.append(spectrogram)
            b.append(dirname.split('/')[-1])
A = np.array(A)
A = A.reshape(A.shape + (1,))

mlb.fit(pd.Series(b).fillna("missing").str.split(', '))
b_mlb = mlb.transform(pd.Series(b).fillna("missing").str.split(', '))
mlb.classes_

A_test = A
b_test = b_mlb

# Testing the model on data with background noise
model.evaluate(A_test, b_test, verbose=1)
b_pred = model.predict(A_test, batch_size=32, verbose=1)

b_pred = (b_pred == b_pred.max(axis=1)[:,None]).astype(int)

print('Model accuracy: ', accuracy_score(b_test, b_pred))

# Metrics
report_df = pd.DataFrame(classification_report(b_test, b_pred, output_dict=True)).transpose()
report_df["label"] = list(mlb.classes_) + ["micro avg","macro avg","weighted avg","samples avg"]
report_df.sort_values(by=['f1-score','support'], ascending=False)

# Extracting self-recorded audio files
patoolib.extract_archive('/content/drive/MyDrive/snimke.zip')

M = []
n = []

for dirname, _, filenames in os.walk('/content/snimke2'):
    for filename in filenames:
        if dirname.split('/')[-1]:
            sample_rate, samples = wavfile.read(os.path.join(dirname, filename))
            frequencies, times, spectrogram = signal.spectrogram(samples, sample_rate)
            M.append(spectrogram)
            n.append(dirname.split('/')[-1])
M = np.array(M)
M = M.reshape(M.shape + (1,))

mlb.fit(pd.Series(n).fillna("missing").str.split(', '))
n_mlb = mlb.transform(pd.Series(n).fillna("missing").str.split(', '))
mlb.classes_

# Testing the model on self-recorded data
M_test = M
n_test = n_mlb

model.evaluate(M_test, n_test, verbose=1)
n_pred = model.predict(M_test, batch_size=32, verbose=1)

n_pred = (n_pred == n_pred.max(axis=1)[:,None]).astype(int)

print('Model accuracy: ', accuracy_score(n_test, n_pred))

# Metrics
report_df = pd.DataFrame(classification_report(n_test, n_pred, output_dict=True)).transpose()
report_df["label"] = list(mlb.classes_) + ["micro avg","macro avg","weighted avg","samples avg"]
report_df.sort_values(by=['f1-score','support'], ascending=False)

# Getting confusion matrix for self-recorded data
num_classes = 10
confusion_matrix_snimke = np.zeros((num_classes, num_classes), dtype=int)

# Calculate confusion matrix
for true, pred in zip(n_test, n_pred):
    for i in range(num_classes):
        if true[i] == 1:
            for j in range(num_classes):
                if pred[j] == 1:
                    confusion_matrix_snimke[i, j] += 1

labels = ['down', 'go', 'left', 'no', 'off', 'on', 'right', 'stop', 'up',
       'yes']
a = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]

# Plot confusion matrix
plt.figure(figsize=(8, 6))
plt.imshow(confusion_matrix_snimke, cmap='Blues', interpolation='nearest')
plt.title('Confusion Matrix')
plt.colorbar()
plt.xlabel('Predicted Label')
plt.ylabel('True Label')
plt.xticks(a, labels, rotation ='vertical')
plt.yticks(a, labels, rotation ='horizontal')
plt.tight_layout()

for i in range(num_classes):
    for j in range(num_classes):
        plt.text(j, i, str(confusion_matrix_snimke[i, j]), ha='center', va='center', color='black')