Adds sr flag for saving responses with specific string

.github/workflows/main.yml

@@ -0,0 +1,33 @@
+# This is a basic workflow to help you get started with Actions
+
+name: CI
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the master branch
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+
+      # Runs a set of commands using the runners shell
+      - name: Run a multi-line script
+        run: |
+          echo Add other actions to build,
+          echo test, and deploy your project.

README.mkd

@@ -1,11 +1,11 @@
 # meg
 
-meg is a tool for fetching lots of URLs but still being 'nice' to servers.
+Created by [Tom Hudson](https://github.com/tomnomnom) (original repo [here](https://github.com/tomnomnom/meg)), meg is a tool for fetching lots of URLs but still being 'nice' to servers.
 
 It can be used to fetch many paths for many hosts; fetching one path
 for all hosts before moving on to the next path and repeating.
 
-You get lots of results quickly, but non of the individual hosts get
+You get lots of results quickly, but none of the individual hosts get
 flooded with traffic.
 
 ## Install
@@ -14,12 +14,9 @@ meg is written in Go and has no run-time dependencies. If you have Go 1.9
 or later installed and configured you can install meg with `go get`:
 
 ```
-▶ go get -u github.com/tomnomnom/meg
+▶ go get -u github.com/GwynHannay/meg
 ```
 
-Or [download a binary](https://github.com/tomnomnom/meg/releases) and
-put it somewhere in your `$PATH` (e.g. in `/usr/bin/`).
-
 ### Install Errors
 
 If you see an error like this it means your version of Go is too old:
@@ -135,13 +132,17 @@ Usage:
   meg [options] [path|pathsFile] [hostsFile] [outputDir]
 
 Options:
-  -c, --concurrency <val>    Set the concurrency level (defaut: 20)
-  -d, --delay <val>          Milliseconds between requests to the same host (default: 5000)
-  -H, --header <header>      Send a custom HTTP header
-  -r, --rawhttp              Use the rawhttp library for requests (experimental)
-  -s, --savestatus <status>  Save only responses with specific status code
-  -v, --verbose              Verbose mode
-  -X, --method <method>      HTTP method (default: GET)
+  -c, --concurrency <val>     Set the concurrency level (defaut: 20)
+  -d, --delay <val>           Milliseconds between requests to the same host (default: 5000)
+  -dr, --discresp <string>    Discard responses containing specific string
+  -H, --header <header>       Send a custom HTTP header
+  -r, --rawhttp               Use the rawhttp library for requests (experimental)
+  -ri, --regexignore <string> Ignore results where the body matches a regex pattern
+  -rk, --regexkeep <string>   Keep results where the body matches a regex pattern
+  -sr, --saveresp <string>    Save only responses containing specific string
+  -s, --savestatus <status>   Save only responses with specific status code
+  -v, --verbose               Verbose mode
+  -X, --method <method>       HTTP method (default: GET)
 
 Defaults:
   pathsFile: ./paths
@@ -233,6 +234,24 @@ use the `-s` or `--savestatus` option:
 ▶ meg --savestatus 200 /robots.txt
 ```
 
+### Saving Only Certain Response Strings
+
+If you only want to save results that contained a certain string, you can
+use the `-sr` or `--saveresp` option:
+
+```
+▶ meg --saveresp '": "v1"' /robots.txt
+```
+
+### Discarding Certain Response Strings
+
+Alternatively, if you don't want to save results that contained a certain string, you can
+use the `-dr` or `--discresp` option:
+
+```
+▶ meg --discresp '": "v1"' /robots.txt
+```
+
 ### Specifying The Method
 
 You can specify which HTTP method to use with the `-X` or `--method` option:

args.go

@@ -44,9 +44,13 @@ type config struct {
 	body           string
 	concurrency    int
 	delay          int
+	discResp       string
 	headers        headerArgs
 	followLocation bool
 	method         string
+	regexIgnore    string
+	regexKeep      string
+	saveResp       string
 	saveStatus     saveStatusArgs
 	timeout        int
 	verbose        bool
@@ -76,6 +80,11 @@ func processArgs() config {
 	flag.IntVar(&delay, "delay", 5000, "")
 	flag.IntVar(&delay, "d", 5000, "")
 
+	// discResp params
+	discResp := ""
+	flag.StringVar(&discResp, "discresp", "", "")
+	flag.StringVar(&discResp, "dr", "", "")
+
 	// headers params
 	var headers headerArgs
 	flag.Var(&headers, "header", "")
@@ -91,6 +100,21 @@ func processArgs() config {
 	flag.StringVar(&method, "method", "GET", "")
 	flag.StringVar(&method, "X", "GET", "")
 
+	// regexIgnore params
+	regexIgnore := ""
+	flag.StringVar(&regexIgnore, "regexIgnore", "", "")
+	flag.StringVar(&regexIgnore, "ri", "", "")
+
+	// regexKeep params
+	regexKeep := ""
+	flag.StringVar(&regexKeep, "regexKeep", "", "")
+	flag.StringVar(&regexKeep, "rk", "", "")
+
+	// saveResp params
+	saveResp := ""
+	flag.StringVar(&saveResp, "saveresp", "", "")
+	flag.StringVar(&saveResp, "sr", "", "")
+
 	// savestatus params
 	var saveStatus saveStatusArgs
 	flag.Var(&saveStatus, "savestatus", "")
@@ -145,9 +169,13 @@ func processArgs() config {
 		body:           body,
 		concurrency:    concurrency,
 		delay:          delay,
+		discResp:       discResp,
 		headers:        headers,
 		followLocation: followLocation,
 		method:         method,
+		regexIgnore:    regexIgnore,
+		regexKeep:      regexKeep,
+		saveResp:       saveResp,
 		saveStatus:     saveStatus,
 		timeout:        timeout,
 		requester:      requesterFn,
@@ -167,16 +195,20 @@ func init() {
 		h += "  meg [path|pathsFile] [hostsFile] [outputDir]\n\n"
 
 		h += "Options:\n"
-		h += "  -b, --body <val>           Set the request body\n"
-		h += "  -c, --concurrency <val>    Set the concurrency level (default: 20)\n"
-		h += "  -d, --delay <millis>       Milliseconds between requests to the same host (default: 5000)\n"
-		h += "  -H, --header <header>      Send a custom HTTP header\n"
-		h += "  -L, --location             Follow redirects / location header\n"
-		h += "  -r, --rawhttp              Use the rawhttp library for requests (experimental)\n"
-		h += "  -s, --savestatus <status>  Save only responses with specific status code\n"
-		h += "  -t, --timeout <millis>     Set the HTTP timeout (default: 10000)\n"
-		h += "  -v, --verbose              Verbose mode\n"
-		h += "  -X, --method <method>      HTTP method (default: GET)\n\n"
+		h += "  -b,  --body <val>           Set the request body\n"
+		h += "  -c,  --concurrency <val>    Set the concurrency level (default: 20)\n"
+		h += "  -d,  --delay <millis>       Milliseconds between requests to the same host (default: 5000)\n"
+		h += "  -dr, --discresp <string>    Discard responses containing specific string\n"
+		h += "  -H,  --header <header>      Send a custom HTTP header\n"
+		h += "  -L,  --location             Follow redirects / location header\n"
+		h += "  -r,  --rawhttp              Use the rawhttp library for requests (experimental)\n"
+		h += "  -ri, --regexignore <string> Ignore results where the body matches a regex pattern\n\n"
+		h += "  -rk, --regexkeep <string>   Keep results where the body matches a regex pattern\n\n"
+		h += "  -sr, --saveresp <string>    Save only responses containing specific string\n"
+		h += "  -s,  --savestatus <status>  Save only responses with specific status code\n"
+		h += "  -t,  --timeout <millis>     Set the HTTP timeout (default: 10000)\n"
+		h += "  -v,  --verbose              Verbose mode\n"
+		h += "  -X,  --method <method>      HTTP method (default: GET)\n\n"
 
 		h += "Defaults:\n"
 		h += "  pathsFile: ./paths\n"

main.go

@@ -6,6 +6,8 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
+	"strings"
 	"sync"
 	"time"
 )
@@ -88,6 +90,29 @@ func main() {
 				continue
 			}
 
+			if len(c.saveResp) > 0 && !(strings.Contains(strings.Join(res.headers, ""), c.saveResp) || (strings.Contains(string(res.body), c.saveResp))) {
+				continue
+			}
+
+			if len(c.discResp) > 0 && (strings.Contains(strings.Join(res.headers, ""), c.discResp) || (strings.Contains(string(res.body), c.discResp))) {
+				continue
+			}
+			if len(c.regexIgnore) > 0 {
+				re := regexp.MustCompile(c.regexIgnore)
+				matched := re.MatchString(res.String())
+				if matched {
+					continue
+				}
+			}
+
+			if len(c.regexKeep) > 0 {
+				re := regexp.MustCompile(c.regexKeep)
+				matched := re.MatchString(res.String())
+				if !matched {
+					continue
+				}
+			}
+
 			if res.err != nil {
 				fmt.Fprintf(os.Stderr, "request failed: %s\n", res.err)
 				continue