resNet50.py

"""
Author: Fergal Stapleton
"""

import sys
import os
import json
import random
import numpy as np
import tensorflow as tf
from keras.models import Sequential
from src import linearGenProgTree
from problem import neuroevolution
# randomGenerationNeuroLGP can be used to test method is better than random,
from algorithm import singleObjNeuroLGP, singleObjNeuroLGPSurrogate, singleObjNeuroLGPBaseline
#from algorithm import randomGenerationNeuroLGP as singleObjNeuroLGP
from sklearn import datasets
from datetime import datetime
from numpy import genfromtxt
#from smt.surrogate_models import KRG, KPLS, KPLSK
#from smt.utils import XSpecs
import time
import timeit
import random
import sys
from scipy.stats import norm
from copy import deepcopy
import pandas as pd
#import psutil
import os
import csv
import heapq
#from memory_profiler import profile
import configparser
import pickle
from keras import backend as K
from tensorflow.keras.datasets import mnist
from tensorflow.keras.utils import to_categorical
from keras import initializers
from keras.models import Sequential, Model
from tensorflow.keras.optimizers import Adam
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau
from tensorflow.keras.applications.resnet50 import ResNet50
from keras.regularizers import l2
from keras.layers import TimeDistributed, Flatten, LSTM, Dense, Activation, ZeroPadding2D, Dropout, BatchNormalization, RepeatVector, Bidirectional
from keras.layers import Conv2D, MaxPooling2D, AveragePooling2D

#physical_devices = tf.config.experimental.list_physical_devices('GPU')
#tf.config.experimental.set_memory_growth(physical_devices[0], True)
#tf.config.experimental.set_memory_growth(physical_devices[1], True)

def read_json(file_path):
    with open(file_path, "r") as f:
        return json.load(f)

def main():
    os.environ['PYTHONHASHSEED'] = str(123)
    random.seed(123)
    tf.random.set_seed(123)
    np.random.seed(123)

  
    multi_gpu = True
    if multi_gpu == True:
	#os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id)
        #physical_devices = tf.config.experimental.list_physical_devices('GPU')
        #tf.config.experimental.set_memory_growth(physical_devices[gpu_id], True)  
        try:
            gpu_id = str(int(sys.argv[1]) % 2)
        except:
            gpu_id = str(0)

        os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id)
        print("CUDA_VISIBLE_DEVICES:", os.environ.get("CUDA_VISIBLE_DEVICES"))

        physical_devices = tf.config.experimental.list_physical_devices('GPU') 
        print(physical_devices)
        
        tf.config.experimental.set_visible_devices(physical_devices[int(gpu_id)], 'GPU')
        tf.config.experimental.set_memory_growth(physical_devices[int(gpu_id)], True)

        print(physical_devices)
        physical_devices2 = tf.config.experimental.list_physical_devices('GPU')
        print(physical_devices2)

        with tf.device('/device:GPU:'+gpu_id):
            if len(sys.argv) > 1:
                run_num = int(sys.argv[1])
            else:
                run_num = 0
        #os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id)


    run_idx = run_num # dumb reassignment
    print("Welcome to GenProgMO library")
    #paramsLoad = read_json('experimentNeuroLGP.json')
    #paramsLoad = read_json('experimentNeuroLGPcatsdogs.json')
    paramsLoad = read_json('experimentNeuroLGPbreakhis200.json')
    print(json.dumps(paramsLoad, indent=4))
    # First and foremost 'what is our problem domain?''
    problem_type = paramsLoad["PROBLEM DOMAIN"]["PROBLEM TYPE"]
    experiment = paramsLoad["EXPERIMENTAL SETUP"]
    surrogate_model_management = paramsLoad["SURROGATE"]
    try:
        problemCode = paramsLoad["PROBLEM DOMAIN"]["PROBLEM CODE"]
        dataSource = paramsLoad["PROBLEM DOMAIN"]["DATA SOURCE"] # Not really needed, but will leave in for consistency or if needed in future
    except:
        print("Error: one of the problem domain properties has not been specified in experiment.json")
        sys.exit(1)
    try:
        opt = paramsLoad["OPTIMIZATION"]
    except:
        print("Error: one of the optimization properties has not been specified in experiment.json")
        sys.exit(1)

    # This is an index that will be associated with these runs. Going forward I will cite the index in GitHub to help with version control
    exp_idx = experiment["EXP ID"]
    experiment_type = experiment["EXPERIMENT TYPE"]
    results_path = experiment["RESULTS PATH"]
    run_start = experiment["RUN START"]
    run_end = experiment["RUN END"]
    dataset = problemCode # TODO repition, unecessary assignment
    run_range = range(int(run_start), int(run_end)) # 0,1,2,...9
    #run = range(10, 20) # 10,11,12,...19

    if experiment_type == "surrogate":
        method = paramsLoad["SURROGATE"]["METHOD"]
    else:
        method = experiment_type

    print("Config settings:")
    print(exp_idx)
    print(experiment_type)
    print(method)
    print(dataset)


    # Set up class files
    gp = linearGenProgTree.GP_tree(paramsLoad) # genetic representation, i.e genetic program
    run = linearGenProgTree.Execute(paramsLoad) # execution
    op = linearGenProgTree.GeneticOP(paramsLoad) #
    probDom = neuroevolution.NeuroLGP(run, gp, gpu_id) # problem domain

    probDom.load_data(dataset, experiment["DATA GENERATOR"])

    #LeNet-5
    lenet = Sequential([
        Conv2D(32, (5, 5), activation='relu', input_shape=(64, 64, 3), kernel_regularizer=l2(0.0001), kernel_initializer=initializers.he_uniform(seed=0)),
        BatchNormalization(momentum = 0.99),
        MaxPooling2D((2, 2)),
        Conv2D(64, (5, 5), activation='relu', kernel_regularizer=l2(0.0001)),
        BatchNormalization(momentum = 0.99),
        MaxPooling2D((2, 2)),
        Flatten(),
        Dense(120, activation='relu'),
        Dropout(0.2, seed=0),
        Dense(84, activation='relu'),
        Dropout(0.2, seed=0),
        Dense(2, activation='softmax')
    ])

    def get_lr_metric(optimizer):
        def lr(y_true, y_pred):
            return optimizer._decayed_lr(tf.float32) # I use ._decayed_lr method instead of .lr
        return lr

    #optimizer = Adam()
    #lr_metric = get_lr_metric(optimizer)

    lenet.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer='adam')

    lenet.summary()

    session = tf.compat.v1.Session()
    K.set_session(session)
    #model.summary()
    #steps_per_epoch=int(self.train_length/self.batch_size),

    epoch_num = experiment["EPOCH NUM"]
    #print(self.train_generator)
    variable_learning_rate = ReduceLROnPlateau(monitor='val_loss', factor = 0.2, patience = 5 , verbose=1)



    lenet.fit(
        probDom.train_generator,
        validation_data=probDom.test_generator,
        epochs=30,
        steps_per_epoch=(probDom.train_length/probDom.batch_size),
        validation_steps=(probDom.test_length/probDom.batch_size),
        verbose = 1,
        callbacks = [variable_learning_rate]
        )
    #acc = model.history.history['val_accuracy'][int(epoch_num - 1)]
    score_train = lenet.evaluate(probDom.train_generator, verbose=0)
    score_test = lenet.evaluate(probDom.test_generator, verbose=0)
    score_val = lenet.evaluate(probDom.val_generator, verbose=0)
    score_val2 = lenet.evaluate(probDom.val2_generator, verbose=0)

    print("LeNet 5")
    print ("%s: %.2f%%" % (lenet.metrics_names[1], score_train[1]*100))
    print ("%s: %.2f%%" % (lenet.metrics_names[1], score_test[1]*100))
    print ("%s: %.2f%%" % (lenet.metrics_names[1], score_val[1]*100))
    print ("%s: %.2f%%" % (lenet.metrics_names[1], score_val2[1]*100))
    
    
    session.close()
    K.clear_session()

    

    vgg16 = Sequential([
        Conv2D(input_shape=(64,64,3),filters=64,kernel_size=(3,3),padding="same", activation="relu", kernel_regularizer=l2(0.0001), kernel_initializer=initializers.he_uniform(seed=0)),
        Conv2D(filters=64,kernel_size=(3,3),padding="same", activation="relu"),
        BatchNormalization(momentum = 0.99),
        MaxPooling2D((2, 2)),
        Conv2D(filters=128, kernel_size=(3,3), padding="same", activation="relu", kernel_regularizer=l2(0.0001)),
        Conv2D(filters=128, kernel_size=(3,3), padding="same", activation="relu", kernel_regularizer=l2(0.0001)),
        BatchNormalization(momentum = 0.99),
        MaxPooling2D((2, 2)),
        Conv2D(filters=256, kernel_size=(3,3), padding="same", activation="relu", kernel_regularizer=l2(0.0001)),
        Conv2D(filters=256, kernel_size=(3,3), padding="same", activation="relu", kernel_regularizer=l2(0.0001)),
        Conv2D(filters=256, kernel_size=(3,3), padding="same", activation="relu", kernel_regularizer=l2(0.0001)),
        BatchNormalization(momentum = 0.99),
        MaxPooling2D((2, 2)),
        Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu", kernel_regularizer=l2(0.0001)),
        Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu", kernel_regularizer=l2(0.0001)),
        Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu", kernel_regularizer=l2(0.0001)),
        BatchNormalization(momentum = 0.99),
        MaxPooling2D((2, 2)),
        Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu", kernel_regularizer=l2(0.0001)),
        Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu", kernel_regularizer=l2(0.0001)),
        Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu", kernel_regularizer=l2(0.0001)),
        BatchNormalization(momentum = 0.99),
        MaxPooling2D((2, 2)),
        Flatten(),
        Dense(units=4096,activation="relu"),
        Dropout(0.2, seed=0),
        Dense(units=4096,activation="relu"),
        Dropout(0.2, seed=0),
        Dense(units=2, activation="softmax")
    ])

    
    vgg16.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer='adam')

    vgg16.summary()

    session = tf.compat.v1.Session()
    K.set_session(session)
    #model.summary()
    #steps_per_epoch=int(self.train_length/self.batch_size),

    epoch_num = experiment["EPOCH NUM"]
    #print(self.train_generator)
    variable_learning_rate = ReduceLROnPlateau(monitor='val_loss', factor = 0.2, patience = 5 , verbose=1)

    vgg16.fit(
        probDom.train_generator,
        validation_data=probDom.test_generator,
        epochs=30,
        steps_per_epoch=(probDom.train_length/probDom.batch_size),
        validation_steps=(probDom.test_length/probDom.batch_size),
        verbose = 1,
        callbacks = [variable_learning_rate]
        )
    #acc = model.history.history['val_accuracy'][int(epoch_num - 1)]
    score_train = vgg16.evaluate(probDom.train_generator, verbose=0)
    score_test = vgg16.evaluate(probDom.test_generator, verbose=0)
    score_val = vgg16.evaluate(probDom.val_generator, verbose=0)
    score_val2 = vgg16.evaluate(probDom.val2_generator, verbose=0)

    print("vgg16")
    print ("%s: %.2f%%" % (vgg16.metrics_names[1], score_train[1]*100))
    print ("%s: %.2f%%" % (vgg16.metrics_names[1], score_test[1]*100))
    print ("%s: %.2f%%" % (vgg16.metrics_names[1], score_val[1]*100))
    print ("%s: %.2f%%" % (vgg16.metrics_names[1], score_val2[1]*100))
    

if __name__ == "__main__":
    # Note: It's not possible to make code deterministic on cuDNN, However seeding has been implemented should one choose to run on cpu's
    # source: https://stackoverflow.com/questions/39938307/determinism-in-tensorflow-gradient-updates
    SEED =123

    #os.environ['TF_GPU_ALLOCATOR'] = 'cuda_malloc_async'
    #TF_GPU_ALLOCATOR=cuda_malloc_async#

    def set_seeds(seed=SEED):
        os.environ['PYTHONHASHSEED'] = str(seed)
        random.seed(seed)
        tf.random.set_seed(seed)
        np.random.seed(seed)

    def set_global_determinism(seed=SEED):

        set_seeds(seed=seed)

        # Below two lines don't work
        os.environ['TF_DETERMINISTIC_OPS'] = '1'
        os.environ['TF_CUDNN_DETERMINISTIC'] = '1'

        # Turn on if using CPU and want deterministic operations
        tf.config.threading.set_inter_op_parallelism_threads(1)
        tf.config.threading.set_intra_op_parallelism_threads(1)

    set_global_determinism(seed=SEED)

    main()