webpyserv.py

#! /usr/bin/python3
# -*- coding: utf-8 -*-

#   Copyright (c) 2017 Ludovic Taffin
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU Affero General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU Affero General Public License for more details.
#
#   You should have received a copy of the GNU Affero General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.

##
#   Tiny Webpy webserver to treat the data from INGInious and using Jplag executable
#   See webpy for more information : http://webpy.org/
#   See Jplag for more information : https://github.com/jplag/jplag
#
##
import datetime
import shutil
import subprocess
import tarfile

import re
import zipfile

import yaml
import web
import os
import io

urls = (
    '/', 'index'
)
t_globals = dict(
    datestr=web.datestr,
)
render = web.template.render('templates/', cache=False,
                             globals=t_globals)
render._keywords['globals']['render'] = render


def verify_file_structure(root_submissions_folder,course_id,task_id):
    course_folder = root_submissions_folder + course_id
    task_folder = course_folder + "/"+task_id
    archive_folder = task_folder + "/ARCHIVES"
    extraction_task_folder =  task_folder + "/EXTRACT"
    if not course_id in os.listdir(root_submissions_folder):
        os.mkdir(course_folder)
    if not task_id in os.listdir(course_folder):
        os.mkdir(task_folder)
    if not "ARCHIVES" in os.listdir(task_folder):
        os.mkdir(archive_folder)
    if not "EXTRACT" in os.listdir(task_folder):
        os.mkdir(extraction_task_folder)
    if not "template" in os.listdir(extraction_task_folder):
        os.mkdir(extraction_task_folder + "/template")

    return archive_folder,extraction_task_folder,course_folder,task_folder
    

class index:
    def GET(self):
        return "Default get response."

    # POST method
    # Return the url to the html page of result.
    def POST(self):
        # Getting data from the form
        input_from_web = web.input()
        taskId = str(input_from_web["taskId"].decode('UTF-8'))
        courseId = str(input_from_web["courseId"].decode('UTF-8'))
        language = input_from_web['lang'].decode('UTF-8')
        percentage = str(input_from_web['percentage'].decode('UTF-8'))
        nyear = str(input_from_web['nyear'].decode('UTF-8'))
        check_archives = str(input_from_web['check_archives'].decode('UTF-8'))
        submissions = input_from_web["sub.tar"]
        exclusion_file = input_from_web["exclude"]
        archive_from_previous_year = input_from_web["archi"]
        template = input_from_web["template"]
        
        root_submissions_folder=os.path.dirname(os.path.realpath(__file__)) + "/soumissions/"
        archivefolder,extractfolder,course_folder,task_folder = verify_file_structure(root_submissions_folder,courseId,taskId)

        def extract_tar_files_from_bytes(source,desti):
            #Extracting submissions to the extract folder
            f= open('sub.tar','wb')
            f.write(source)
            f.close()
            file_like_object = io.BytesIO(source)
            ar = tarfile.open(fileobj=file_like_object)
            ar.extractall(path=desti)
            ar.close()
        
        #Extracting submissions to the extract folder
        extract_tar_files_from_bytes(submissions,extractfolder)
        #Fill in template folder
        if template:
            extract_tar_files_from_bytes(template,extractfolder+"/template")
        # Getting old and added submissions in the extract
        if archive_from_previous_year:
            extract_tar_files_from_bytes(archive_from_previous_year,extractfolder)
        
        #Save exclude file
        with open(task_folder+"/exclude.txt", 'wb+') as file:
            file.write(exclusion_file)


        #check if submissions are zip file or code files
        for root, subdirs, files in os.walk(extractfolder):
            if (root.endswith("uploaded_files")):
                for elem in files:
                    if elem.endswith("zip"):
                        myzip = zipfile.ZipFile(root + "/" + elem)
                        try:
                            for f in myzip.namelist():
                                myzip.open(f)
                                myzip.extract(f, path=root)
                        except zipfile.BadZipfile:
                            print("exception")
                        myzip.close()
                    elif elem.endswith(".tar.gz") or elem.endswith(".tgz"):
                        mytar = tarfile.open(root+"/"+elem)
                        mytar.extractall(path=root)
                        mytar.close()
                    else:
                        pass
        
        if(check_archives=="on"):
            # Compare submission of this year to archived one
            for sub in os.listdir(archivefolder):
                old_name = archivefolder+"/"+sub
                new_name = extractfolder+"/archive_"+sub
                try:
                    os.mkdir(new_name)
                    shutil.copytree(old_name,new_name)
                except FileExistsError:
                    pass

        ##EXECUTION##
        with open("config.yaml", 'r') as stream:
            ls = yaml.load(stream, Loader=yaml.CLoader)
            loc = ls['jplag']['localisation']
            nameOfJar = ls['jplag']['name']
            completeloc = loc + "/" + nameOfJar
            f = open('error.txt', "wb")

            subprocess.call(['java', '-jar',
                             completeloc,
                             '-m', percentage + '%',
                             '-bc', "template/",
                             '-x', task_folder+"/exclude.txt",
                             '-l', language,
                             '-r', os.path.dirname(os.path.realpath(__file__)) + '/static/result/' + courseId + "/" + taskId,
                             '-s', extractfolder], stdout=f)
            f.close()
            with open('error.txt', 'r') as f2:
                for line in f2.readlines():
                    if line.startswith("Error"):
                        shutil.rmtree(extractfolder, ignore_errors=True)
                        return web.ctx['homedomain'] + '/static/error.html'

        if(nyear=="on"):
            # Store submissions of this submission for future jplag test
            its = int(datetime.datetime.now().timestamp())
            for sub in os.listdir(extractfolder):
                if not sub.startswith("/archive_"):
                    new_name = archivefolder+"/"+str(its)+"_"+sub
                    old_name = extractfolder+"/"+sub
                    try:
                        os.mkdir(new_name)
                    except FileExistsError:
                        pass
                    try:
                        os.rename(old_name,new_name)
                    except FileNotFoundError:
                        pass

        shutil.rmtree(extractfolder, ignore_errors=True)
        return web.ctx['homedomain'] + '/static/result/' + courseId + "/" + taskId

app = web.application(urls, globals())
wsgiapp = app.wsgifunc()

# Standard running script
if __name__ == "__main__":
    app.run()