Cartographer and PixPlot Image preview

.gitignore

@@ -44,14 +44,6 @@ webtool/venv/
 *.ipynb
 venv/
 
-# do not ignore interface images
-!webtool/static/img/*.png
-!webtool/static/img/*.gif
-!webtool/static/img/*.jpg
-!webtool/static/img/favicon/*.ico
-!webtool/static/img/flags/*.png
-!common/assets/github-screenshots/*.png
-
 # generated by 4CAT
 webtool/static/css/colours.css
 webtool/static/img/favicon/favicon.ico
@@ -66,3 +58,13 @@ keys/
 images/
 sphinx-3.3.1/
 sphinx/
+
+# do not ignore interface images
+!webtool/static/img/*.png
+!webtool/static/img/*.gif
+!webtool/static/img/*.jpg
+!webtool/static/img/favicon/*.ico
+!webtool/static/img/flags/*.png
+!webtool/static/pixplot_template/assets/images/*
+!webtool/static/pixplot_template/assets/images/icons/*
+!common/assets/github-screenshots/*.png

backend/lib/processor.py

@@ -662,7 +662,7 @@ def write_csv_items_and_finish(self, data):
         self.dataset.update_status("Finished")
         self.dataset.finish(len(data))
 
-    def write_archive_and_finish(self, files, num_items=None, compression=zipfile.ZIP_STORED, finish=True):
+    def write_archive_and_finish(self, filelist_or_folder, num_items=None, compression=zipfile.ZIP_STORED, finish=True):
         """
         Archive a bunch of files into a zip archive and finish processing
 
@@ -676,21 +676,28 @@ def write_archive_and_finish(self, files, num_items=None, compression=zipfile.ZI
         :param bool finish:  Finish the dataset/job afterwards or not?
         """
         is_folder = False
-        if issubclass(type(files), PurePath):
-            is_folder = files
-            if not files.exists() or not files.is_dir():
-                raise RuntimeError("Folder %s is not a folder that can be archived" % files)
+        if issubclass(type(filelist_or_folder), PurePath):
+			# folder with files
+            is_folder = filelist_or_folder
+            if not filelist_or_folder.exists() or not filelist_or_folder.is_dir():
+                raise RuntimeError("Folder %s is not a folder that can be archived" % filelist_or_folder)
 
-            files = files.glob("*")
-
-        # create zip of archive and delete temporary files and folder
+		# create zip of archive and delete temporary files and folder
         self.dataset.update_status("Compressing results into archive")
         done = 0
-        with zipfile.ZipFile(self.dataset.get_results_path(), "w", compression=compression) as zip:
-            for output_path in files:
-                zip.write(output_path, output_path.name)
-                output_path.unlink()
-                done += 1
+        with zipfile.ZipFile(self.dataset.get_results_path(), "w", compression=compression) as zipf:
+            if is_folder:
+                for root, dirs, files in os.walk(filelist_or_folder):
+                    for file in files:
+                        zipf.write(os.path.join(root, file),
+                                    os.path.relpath(os.path.join(root, file), filelist_or_folder))
+                        done += 1
+            else:
+                # list of files
+                for output_path in filelist_or_folder:
+                    zipf.write(output_path, output_path.name)
+                    output_path.unlink()
+                    done += 1
 
         # delete temporary folder
         if is_folder:

backend/workers/cleanup_tempfiles.py

@@ -63,10 +63,13 @@ def work(self):
             # if for whatever reason there are multiple hashes in the filename,
             # the key would always be the last one
             key = possible_keys.pop()
-
             try:
                 dataset = DataSet(key=key, db=self.db)
             except DataSetException:
+                # TODO: would another dataset user same get_result_folder_path? and additional files?
+                if self.db.fetchone(f"select * from datasets where result_file = '{file.name}'") is not None:
+                    # Another dataset is using this file
+                    continue
                 # the dataset has been deleted since, but the result file still
                 # exists - should be safe to clean up
                 if file.name not in tracked_files:

common/lib/dataset.py

@@ -891,20 +891,21 @@ def delete(self, commit=True, queue=None):
         self.db.delete("datasets_owners", where={"key": self.key}, commit=commit)
         self.db.delete("users_favourites", where={"key": self.key}, commit=commit)
 
-        # delete from drive
-        try:
-            if self.get_results_path().exists():
-                self.get_results_path().unlink()
-            if self.get_results_path().with_suffix(".log").exists():
-                self.get_results_path().with_suffix(".log").unlink()
-            if self.get_results_folder_path().exists():
-                shutil.rmtree(self.get_results_folder_path())
-
-        except FileNotFoundError:
-            # already deleted, apparently
-            pass
-        except PermissionError as e:
-            self.db.log.error(
+        # delete from drive if not used elsewhere
+        if self.db.fetchone(f"select * from datasets where result_file = '{self.get_results_path().name}' and key != '{self.key}'") is None:
+            try:
+                if self.get_results_path().exists():
+                    self.get_results_path().unlink()
+                if self.get_results_path().with_suffix(".log").exists():
+                    self.get_results_path().with_suffix(".log").unlink()
+                if self.get_results_folder_path().exists():
+                    shutil.rmtree(self.get_results_folder_path())
+
+            except FileNotFoundError:
+                    # already deleted, apparently
+                    pass
+            except PermissionError as e:
+                self.db.log.error(
                 f"Could not delete all dataset {self.key} files; they may need to be deleted manually: {e}"
             )
 

common/lib/helpers.py

@@ -2,6 +2,7 @@
 Miscellaneous helper functions for the 4CAT backend
 """
 import subprocess
+import zipfile
 import imagehash
 import hashlib
 import requests
@@ -119,6 +120,7 @@ def sniff_encoding(file):
 
     return "utf-8-sig" if maybe_bom == b"\xef\xbb\xbf" else "utf-8"
 
+
 def sniff_csv_dialect(csv_input):
     """
     Determine CSV dialect for an input stream
@@ -141,6 +143,13 @@ def sniff_csv_dialect(csv_input):
     return dialect, has_header
 
 
+def get_html_redirect_page(url):
+    """
+    Returns a html string to redirect to PixPlot.
+    """
+    return f"<head><meta http-equiv='refresh' charset='utf-8' content='0; URL={url}'></head>"
+
+
 def get_git_branch():
     """
     Get current git branch
@@ -1279,6 +1288,22 @@ def split_urls(url_string, allowed_schemes=None):
     return potential_urls
 
 
+def get_archived_file(archive_path, archived_file, temp_dir):
+    with zipfile.ZipFile(archive_path, "r") as archive_file:
+        archive_contents = sorted(archive_file.namelist())
+
+        if archived_file in archive_contents:
+            info = archive_file.getinfo(archived_file)
+            if info.is_dir():
+                raise IsADirectoryError("File is a directory")
+
+            archive_file.extract(archived_file, temp_dir)
+
+            return temp_dir.joinpath(archived_file)
+
+        else:
+            raise FileNotFoundError("File not found in archive")
+
 def folder_size(path='.'):
     """
     Get the size of a folder using os.scandir for efficiency

processors/machine_learning/pix-plot.py

@@ -11,7 +11,7 @@
 from werkzeug.utils import secure_filename
 
 from common.lib.dmi_service_manager import DmiServiceManager, DsmOutOfMemory, DmiServiceManagerException
-from common.lib.helpers import UserInput, ellipsiate
+from common.lib.helpers import UserInput, get_html_redirect_page, ellipsiate
 from backend.lib.processor import BasicProcessor
 
 __author__ = "Dale Wahl"
@@ -229,7 +229,7 @@ def process(self):
 
         # Results HTML file redirects to output_dir/index.html
         plot_url = ('https://' if self.config.get("flask.https") else 'http://') + self.config.get("flask.server_name") + '/result/' + f"{os.path.relpath(self.dataset.get_results_folder_path(), self.dataset.folder)}/index.html"
-        html_file = self.get_html_page(plot_url)
+        html_file = get_html_redirect_page(plot_url)
 
         # Write HTML file
         with self.dataset.get_results_path().open("w", encoding="utf-8") as output_file:
@@ -387,12 +387,6 @@ def format_metadata(self, temp_path):
         self.dataset.update_status("Metadata.csv created")
         return metadata_file_path if rows_written != 0 else False
 
-    def get_html_page(self, url):
-        """
-        Returns a html string to redirect to PixPlot.
-        """
-        return f"<head><meta http-equiv='refresh' charset='utf-8' content='0; URL={url}'></head>"
-
     def clean_filename(self, s):
         """
         Given a string that points to a filename, return a clean filename

processors/networks/coordinate_map.py

@@ -0,0 +1,53 @@
+import json
+
+from backend.lib.processor import BasicProcessor
+
+
+__author__ = "Dale Wahl"
+__credits__ = ["Dale Wahl"]
+__maintainer__ = "Dale Wahl"
+__email__ = "4cat@oilab.eu"
+
+from common.lib.user_input import UserInput
+
+
+class CoordinateMap(BasicProcessor):
+    """
+    Wrapper DataSet for a JSON file with plot coordinates that can be used by the cartographer to plot images accordingly
+    """
+    type = "coordinate-map"  # job type ID
+    category = "Networks"  # category
+    title = "Coordinate Map"  # title displayed in UI
+    description = "Generate via network \"preview\" and export node coordinates"  # description displayed in UI
+    extension = "json"  # extension of result file, used internally and in UI
+
+    options= {
+        "coordinates": {
+            "type": UserInput.OPTION_TEXT_JSON,
+            "default": {},
+            "help": "JSON containing nodes and their coordinates",
+            "tooltip": "e.g. {'node_1': {'x': 0, 'y': 0}, 'node_2': {'x': 1, 'y': 1}}",
+        }
+    }
+
+    @classmethod
+    def is_compatible_with(cls, module=None, config=None):
+        """
+        Currently can only be used by the sigma network visualizer; no 4CAT modules have the appropriate input
+        """
+        # TODO: this needs to be able to run on network datasets, but requires input from sigma preview
+        # How do we hide from frontend, but still allow is_compatible_with to return True?
+        # return module.get_extension() == "gexf"
+        return False # TODO enable this, button in gexf.html when notifications work...
+
+    def process(self):
+        """
+        This takes a JSON  containing coordinates as input and saves it as a 4CAT Dataset. Designed to be used with
+        the sigma network visualizer.
+        """
+        json_data = self.parameters.get("coordinates")
+
+        with open(self.dataset.get_results_path(), "w") as f:
+            f.write(json.dumps(json_data))
+
+        self.dataset.finish(len(json_data))