[FIX] script crashed when data directory its not present

[FIX] Script crashed when SVG data its corrupted or incomplete, for now it bypass the file.
[FIX] Script crashed when map directory its not present
[UPD] improved the algorithm to download the maps to retrie the download if an error happened until 3 retries.

Author: rafaga

database_builder.py

@@ -55,6 +55,8 @@ def update_as_needed(resource_name):
     downloaded_data = download_control(md5_url)
 
     if downloaded_data is not None:
+        if not Path(".").joinpath("data").exists():
+            Path(".").joinpath("data").mkdir()
         shutil.move(Path('.').joinpath(files[0]),md5_file)
         with open(md5_file, 'rt', encoding="UTF-8") as file:
             md5.append(file.read())

external_parser.py

@@ -373,10 +373,14 @@ def _extract_map_data(self, map_filename):
         map_file = Path(map_filename)
         if not map_file.exists():
             print("file not found ... skiping parsing")
-            return
+            return False
 
         e_tree = xml.etree.ElementTree.ElementTree()
-        root = e_tree.parse(source=map_filename)
+        try:
+            root = e_tree.parse(source=map_filename)
+        except xml.etree.ElementTree.ParseError as err:
+            print("Error on parsing " + map_filename + " - " + err.msg)
+            return False
         solar_system_ids = []
 
         # here we are iterating over Ice Belts and adding to Database
@@ -409,6 +413,7 @@ def _extract_map_data(self, map_filename):
             cur.execute(query, solar_systems)
         cur.close()
         cur.connection.commit()
+        return True
 
     def get_all_regions(self):
         """Function to get all regions from SDE and download the svg maps from dotlan"""
@@ -448,15 +453,23 @@ def process(self):
         for region in eve_regions:
             file_size = 0
             map_filepath = Path(self.data_directory).joinpath('maps').joinpath(str(region[1]).replace(' ', '_') + '.svg')
-            if not map_filepath.exists():
-                map_url = self.map_url + region[1].replace(' ', '_') + ".svg"
-                urlparse(map_url)
-                file_size = MiscUtils.download_file(map_url, str(region[1]).replace(' ', '_') + ".svg")
-                if file_size is None or file_size <= 100:
+            for download_try in range(1,4):
+                if not map_filepath.exists():
+                    map_url = self.map_url + region[1].replace(' ', '_') + ".svg"
+                    urlparse(map_url)
+                    file_size = MiscUtils.download_file(map_url, str(region[1]).replace(' ', '_') + ".svg")
+                    if file_size is None or file_size <= 100:
+                        map_filepath.unlink()
+                        print("Dotlan: Invalid data was recieved for " + region[1])
+                    else:
+                        if not Path(self.data_directory).joinpath('maps').exists():
+                            Path(self.data_directory).joinpath('maps').mkdir()
+                        shutil.move( str(region[1]).replace(' ', '_') + ".svg",map_filepath)
+                        print("Dotlan: Downloaded Map for " + region[1])
+                print("Dotlan: parsing data for " + region[1])
+                if not self._extract_map_data(map_filepath):
+                    download_try += 1
                     map_filepath.unlink()
-                    print("Dotlan: Invalid data was recieved for " + region[1])
+                    print("Dotlan: Invalid data for " + region[1] + ", retrying download (" + download_try + ").")
                 else:
-                    shutil.move( str(region[1]).replace(' ', '_') + ".svg",map_filepath)
-                    print("Dotlan: Downloaded Map for " + region[1])
-            print("Dotlan: parsing data for " + region[1])
-            self._extract_map_data(map_filepath)
+                    break