PTools changes

.gitignore

@@ -1 +1,2 @@
 *.pyc
+PAV/scripts/site

PAV/modules/basejobcontroller.py

@@ -70,6 +70,8 @@
 from subprocess import Popen, PIPE
 import subprocess
 
+from testConfig import decode_metavalue
+
 
 def copy_file(src, dest):
     try:
@@ -82,8 +84,17 @@ def copy_file(src, dest):
         print('Error: %s' % e.strerror)
 
 
-class JobController:
 
+class JobException(Exception):
+    def __init__(self, exit_code, output):
+        super(JobException, self).__init__(output)
+        self.err_code = exit_code
+        self.err_str = os.strerror(exit_code)
+        self.msg = output
+
+
+
+class JobController(object):
     """ class to define the common actions for any job type """
 
     @staticmethod
@@ -118,7 +129,7 @@ def __init__(self, uid, configs, job_log_file):
         is_exec = os.access(mycmd, os.X_OK)
         if not is_exec:
             print mycmd + " Run command not executable!"
-            self.logger.error('%s %s not executable' % (self.lh + ":", mycmd))
+            self.logger.error(self.lh + ": " + mycmd + " not executable")
             raise RuntimeError('Run command not executable')
 
         self.logger.info(self.lh + " : init phase")
@@ -132,7 +143,7 @@ def __init__(self, uid, configs, job_log_file):
         os.environ['GZ_TESTNAME'] = self.name
         os.environ['PV_TESTEXEC'] = self.configs['run']['cmd']
         os.environ['GZ_TESTEXEC'] = self.configs['run']['cmd']
-        pt = type(self.configs['run']['test_args'])
+        #pt = type(self.configs['run']['test_args'])
         try:
             # ++ PV_TEST_ARGS : Test arguments for job extracted from test suite
             os.environ['GZ_TEST_PARAMS'] = self.configs['run']['test_args']
@@ -170,11 +181,14 @@ def setup_working_space(self):
             return
 
         # now setup and do the move
-        os.environ['PV_RUNHOME'] = ws + "/" + self.name + "__" + run_cmd.split("/",1)[0] + "." + JobController.now()
+        os.environ['PV_RUNHOME'] = ws + "/" + self.name + "__" + \
+                                   run_cmd.split("/", 1)[0] + "." + \
+                                   JobController.now()
 
         print 'Working Space: %s' % os.environ['PV_RUNHOME']
 
-        self.logger.info(self.lh + " : " + 'Create temporary Working Space - ' + os.environ['PV_RUNHOME'])
+        self.logger.info(self.lh + " : " + 'Create temporary Working Space - '
+                         + os.environ['PV_RUNHOME'])
         try:
             os.makedirs(os.environ['PV_RUNHOME'], 0o775)
         except OSError:
@@ -194,12 +208,14 @@ def setup_working_space(self):
             from_loc = src_dir + "/"
             if exclude_ws:
                 cmd = "rsync -a --exclude '" + \
-                      exclude_ws + "' --exclude '*.[ocfh]' --exclude '*.bck' --exclude '*.tar' "
+                      exclude_ws + "' --exclude '*.[ocfh]' --exclude" + \
+                      " '*.bck' --exclude '*.tar' "
             else:
-                cmd = "rsync -a --exclude '*.[ocfh]' --exclude 'pv_ws' --exclude '*.bck' --exclude '*.tar' "
+                cmd = "rsync -a --exclude '*.[ocfh]' --exclude 'pv_ws'" + \
+                      " --exclude '*.bck' --exclude '*.tar' "
             cmd += from_loc + " " + to_loc
 
-        self.logger.debug('%s : %s' % (self.lh, cmd))
+        self.logger.debug(self.lh + " : " + cmd)
 
         # run the command
         p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
@@ -208,8 +224,9 @@ def setup_working_space(self):
         if p.returncode or errors:
             print "Error: failed copying data to working space!"
             print [p.returncode, errors, output]
-            self.logger.info(self.lh + " failed copying data to working space!, skipping job: " + self.name +
-                                       "(Hint: check the job logfile)")
+            self.logger.info(self.lh + " failed copying data to working space!"
+                             + "skipping job: " + self.name +
+                             "(Hint: check the job logfile)")
             # self.logger.info(self.lh + p.returncode + errors + output)
 
     def __str__(self):
@@ -272,10 +289,9 @@ def run_epilog():
                 print "- Run epilog script: " + str(es)
                 os.system(es)
                 print "- epilog script complete"
-        except KeyError, e:
-                # print 'I got a KeyError - no: "%s"' % str(e)
-                print "- No epilog script configured"
-                pass
+        except KeyError:
+            # print 'I got a KeyError - no: "%s"' % str(e)
+            print "- No epilog script configured"
 
     def setup_job_info(self):
 
@@ -297,10 +313,10 @@ def setup_job_info(self):
         # Support for a Splunk data log or file
         try:
             if self.configs['splunk']['state']:
-                os.environ['SPLUNK_GDL'] = str(self.configs['splunk']['global_data_file'])
+                os.environ['SPLUNK_GDL'] = decode_metavalue(self.configs['splunk']['global_data_file'])
+                print '<global data file> ' + os.environ['SPLUNK_GDL']
         except KeyError, e:
             print 'basejobcontroller:setup_job_info, Splunk config error - no: "%s"' % str(e)
-            pass
 
     @staticmethod
     def cleanup():
@@ -332,10 +348,8 @@ def cleanup():
                             copy_file(file2save, to_loc)
         except KeyError, e:
             print 'I got a KeyError -  "%s"' % str(e)
-            pass
         except:
             print 'Warning!, copy failed!'
-            pass
 
         # remove the working space ONLY if it was created
         try:
@@ -367,7 +381,7 @@ def generate_trend_data_file(cls):
         for line in lf:
             # td_regex = os.environ['TD_REGX']
             # match = re.search(td_regex, line, re.IGNORECASE)
-            match_str = "(<td>\s+(.*))"
+            match_str = r"(<td>\s+(.*))"
             match = re.search(match_str, line, re.IGNORECASE)
             # match = re.search("^(<td>\s+(.*))", line, re.IGNORECASE)
             if match:
@@ -391,7 +405,7 @@ def process_trend_data():
                 log_dir = os.environ['PV_JOB_RESULTS_LOG_DIR']
                 cmd = os.environ['PVINSTALL'] + "/PAV/scripts/splunk/td2splunkData " + log_dir
                 os.system(cmd)
-        except KeyError, e:
+        except KeyError:
             # Never set up properly so just move on...
             # print 'basejobcontroller:process_trend_data, KeyError - no: "%s"' % str(e)
             pass

PAV/modules/helperutilities.py

@@ -63,7 +63,7 @@ def which(program):
     def is_exe(fpath):
         return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
 
-    fpath, fname = os.path.split(program)
+    fpath, _ = os.path.split(program)
     if fpath:
         if is_exe(program):
             return program

PAV/modules/ldms.py

@@ -63,12 +63,12 @@
 import logging
 
 
-class LDMS():
+class LDMS(object):
 
     def __init__(self, my_te):
         my_name = self.__class__.__name__
         self.logger = logging.getLogger('pav.' + my_name)
-        self.logger.info('initialize %s to run: ' % my_name)
+        self.logger.info('initialize ' + my_name + ' to run: ')
 
         self.pid = str(os.getpid())
         self.lh = my_name
@@ -104,7 +104,6 @@ def create_output_dir(self):
             print " Error creating metrics directory : \n\t" + output_dir
             self.logger.info(self.lh + " Error creating metrics directory : \n\t" + output_dir)
             output_dir = ''
-            pass
 
         print "Created ldms metrics dir: " + output_dir
 
@@ -128,14 +127,14 @@ def start():
 
         text_file = open(outfile, "w")
         try:
-            subprocess.Popen(os.environ['LDMS_START_CMD'], stdout=text_file, stdin=open(os.devnull), shell=True)
+            subprocess.Popen(os.environ['LDMS_START_CMD'], stdout=text_file,
+                             stdin=open(os.devnull), shell=True)
         except subprocess.CalledProcessError as e:
             ret = e.returncode
             if ret in (1, 2):
                 print("the command failed")
             elif ret in (3, 4, 5):
                 print("the command failed very much")
-            pass
 
     @staticmethod
     def status(jid):