Update 0.5

- scraped pastes now are automatically sorted into subdirectories unless the noSorting flag is set

Author: InitialPosition

.gitignore

@@ -139,3 +139,6 @@ keywords.txt
 
 # pycharm stuff
 .idea/
+
+# vscode
+.vscode/

README.md

@@ -18,8 +18,9 @@ Parameters are back baby
 | Parameter | Optional? | Default | Effect |
 | --------- | --------- | ------- | ------ |
 | -k / --keywords | Yes | None | A file containing keywords to scrape for. One keyword per line. |
-| -i / --infinite | Yes | No | Whether to run in infinite mode or in keyfile mode |
+| -i / --infinite | Yes | False | Whether to run in infinite mode or in keyfile mode |
+| -nS / --noSorting | Yes | False | Whether to sort keyword pastes into subdirectories |
 
 ### Roadmap
 - [X] Parameter input
-- [ ] Index file for keywords
+- [X] Index for keywords

scrape.py

@@ -76,13 +76,17 @@ def main():
                                                                                                        keyword),
                                                                               "green")
 
+                        if args.noSorting is False:
+                            path_file = path.join("files", keyword, "{0}.txt".format(entry["key"]))
+
                         with open(path_file, "w+", encoding='utf-8') as entry_file:
                             entry_file.write(entry_content)
 
                         break
             else:
                 with open(path_file, "w+", encoding='utf-8') as entry_file:
                     entry_file.write(entry_content)
+
                 bar.suffix = "%(index)d/%(max)d Saving paste \'{0}\'".format(entry["key"])
 
             bar.next()
@@ -113,7 +117,7 @@ def main():
 
     AUTHOR = "SYRAPT0R"
     COPYRIGHT = "2019-2020"
-    VERSION = "0.4.5"
+    VERSION = "0.5"
 
     # parse arguments
     keywords = None
@@ -123,6 +127,8 @@ def main():
     parser.add_argument("-k", "--keywords", help="A file containing keywords for the search")
     parser.add_argument("-i", "--infinite", help="Whether to run in infinite mode (Default: false)",
                         action="store_true", default=False)
+    parser.add_argument("-nS", "--noSorting", help="Whether to sort keyword pastes into subdirectories",
+                        action="store_true", default=False)
 
     args = parser.parse_args()
 
@@ -141,11 +147,18 @@ def main():
 
         except IOError:
             status(termcolor.colored("Unable to load specified keyword file. Aborting...", "red"))
-
             exit(0)
 
         keywords = [keyword.strip() for keyword in keywords]
 
+        # create subdirectories if required
+        if args.noSorting is False:
+            for keyword in keywords:
+                current_path = path.join("files", keyword)
+                if not path.isdir(current_path):
+                    status(termcolor.colored("Creating directory {0}".format(current_path), "yellow"))
+                    mkdir(current_path)
+
         status("Loaded {0} keywords".format(len(keywords)))
 
     # create paste ID index