dataAnalysis.py

"""
Author: Fergal Stapleton
"""

import sys
import os
import json
import random
import numpy as np
import tensorflow as tf
from src import linearGenProgTree
from problem import neuroevolution
# randomGenerationNeuroLGP can be used to test method is better than random,
from algorithm import dataLoad
#from algorithm import randomGenerationNeuroLGP as singleObjNeuroLGP
from sklearn import datasets
from datetime import datetime
from numpy import genfromtxt
#from smt.surrogate_models import KRG, KPLS, KPLSK
#from smt.utils import XSpecs
import time
import timeit
import random
import sys
from scipy.stats import norm
from copy import deepcopy
import pandas as pd
#import psutil
import os
import csv
import heapq
#from memory_profiler import profile
import configparser
import pickle

#physical_devices = tf.config.experimental.list_physical_devices('GPU')
#tf.config.experimental.set_memory_growth(physical_devices[0], True)
#tf.config.experimental.set_memory_growth(physical_devices[1], True)

def read_json(file_path):
    with open(file_path, "r") as f:
        return json.load(f)

def main():
    print("Welcome to GenProgMO library")
    #paramsLoad = read_json('experimentNeuroLGP.json')
    #paramsLoad = read_json('experimentNeuroLGPcatsdogs.json')
    #paramsLoad = read_json('experimentNeuroLGPmnistevenandodd.json')
    paramsLoad = read_json('experimentNeuroLGPbreakhis200.json')
    #paramsLoad = read_json('experimentNeuroLGPfashionmnistfootwear.json')
    print(json.dumps(paramsLoad, indent=4))
    # First and foremost 'what is our problem domain?''
    problem_type = paramsLoad["PROBLEM DOMAIN"]["PROBLEM TYPE"]
    experiment = paramsLoad["EXPERIMENTAL SETUP"]
    surrogate_model_management = paramsLoad["SURROGATE"]
    try:
        problemCode = paramsLoad["PROBLEM DOMAIN"]["PROBLEM CODE"]
        dataSource = paramsLoad["PROBLEM DOMAIN"]["DATA SOURCE"] # Not really needed, but will leave in for consistency or if needed in future
    except:
        print("Error: one of the problem domain properties has not been specified in experiment.json")
        sys.exit(1)
    try:
        opt = paramsLoad["OPTIMIZATION"]
    except:
        print("Error: one of the optimization properties has not been specified in experiment.json")
        sys.exit(1)

    # This is an index that will be associated with these runs. Going forward I will cite the index in GitHub to help with version control
    exp_idx = experiment["EXP ID"]
    experiment_type = experiment["EXPERIMENT TYPE"]
    results_path = experiment["RESULTS PATH"]
    run_start = experiment["RUN START"]
    run_end = experiment["RUN END"]
    dataset = problemCode # TODO repition, unecessary assignment
    run_range = range(int(run_start), int(run_end)) # 0,1,2,...9
    #run = range(10, 20) # 10,11,12,...19

    if experiment_type == "surrogate":
        method = paramsLoad["SURROGATE"]["METHOD"]
    else:
        method = experiment_type

    print("Config settings:")
    print(exp_idx)
    print(experiment_type)
    print(method)
    print(dataset)


    # Set up class files
    gp = linearGenProgTree.GP_tree(paramsLoad) # genetic representation, i.e genetic program
    run = linearGenProgTree.Execute(paramsLoad) # execution
    op = linearGenProgTree.GeneticOP(paramsLoad) #
    probDom = neuroevolution.NeuroLGP(run, gp) # problem domain

    
    # # actually doesn't matter what we load here, just want to access the data loaders
    alg = dataLoad.Standard(probDom, gp, run, op, experiment, opt, problemCode)  
    alg.loader(method, dataset, exp_idx)


    #print(alg.dataGenerator)
    print(alg.pd.train_generator)
    print(alg.pd.test_generator)
    print(alg.pd.val_generator)
    print(alg.pd.batch_size) 

    # Initialize empty lists to store data
    all_x = []
    all_y = []

    # Iterate through the generator and accumulate data
    for batch_x, batch_y in alg.pd.train_generator:
        all_x.append(batch_x)
        all_y.append(batch_y)
        
        # Check for end of generator
        if len(all_x) * alg.pd.batch_size >= alg.pd.train_length:
            break

    # Concatenate accumulated data into numpy arrays
    alg.pd.X_train = np.concatenate(all_x)
    alg.pd.y_train = np.concatenate(all_y)

    print(alg.pd.X_train.shape)
    print(alg.pd.y_train.shape)
    #print(alg.pd.Y_test)
    #print(alg.pd.X_test)
    #print(alg.pd.X_val) 
    #print(alg.pd.Y_val)


if __name__ == "__main__":
    # Note: It's not possible to make code deterministic on cuDNN, However seeding has been implemented should one choose to run on cpu's
    # source: https://stackoverflow.com/questions/39938307/determinism-in-tensorflow-gradient-updates
    SEED =123

    #os.environ['TF_GPU_ALLOCATOR'] = 'cuda_malloc_async'
    #TF_GPU_ALLOCATOR=cuda_malloc_async#

    def set_seeds(seed=SEED):
        os.environ['PYTHONHASHSEED'] = str(seed)
        random.seed(seed)
        tf.random.set_seed(seed)
        np.random.seed(seed)

    def set_global_determinism(seed=SEED):

        set_seeds(seed=seed)

        # Below two lines don't work
        #os.environ['TF_DETERMINISTIC_OPS'] = '1'
        #os.environ['TF_CUDNN_DETERMINISTIC'] = '1'

        # Turn on if using CPU and want deterministic operations
        #tf.config.threading.set_inter_op_parallelism_threads(1)
        #tf.config.threading.set_intra_op_parallelism_threads(1)

    set_global_determinism(seed=SEED)

    main()