URL PatternFiltering.md

package main

import (
  "encoding/json"
  "flag"
  "fmt"
  "io/ioutil"
  "net/http"
  "os"
  "strings"
)

func main() {
  jsonUrl := flag.String("jsonUrl", "https://gist.githubusercontent.com/0x3n0/fa105b397c3b367c3203927af987b2d6/raw/7bd4c313d4939dc58d2700056a90fd8150efab9f/patterns.json", "URL of json file")
  cutFlag := flag.Bool("cut", false, "Remove query parameters from URLs")
  getFlag := flag.String("get", "", "Filter URLs by parameters")
  flag.Parse()

  if len(os.Args) > 1 && (os.Args[1] == "-h" || os.Args[1] == "--help") {
    fmt.Println("Usage: go run script.go [-jsonUrl=<json_url>] [-cut] [-get=<parameters>]")
    fmt.Println("The [-jsonUrl=<json_url>] option is used to specify the URL of the json file containing the patterns")
    fmt.Println("The [-cut] option is used to remove the query parameters from the URLs.")
    fmt.Println("The [-get=<parameters>] option is used to filter URLs by parameters.")
    return
  }
  
  input, _ := ioutil.ReadAll(os.Stdin)
  output := string(input)

  resp, err := http.Get(*jsonUrl)
  if err != nil {
    fmt.Println(err)
    os.Exit(1)
  }

  defer resp.Body.Close()
  byteValue, _ := ioutil.ReadAll(resp.Body)

  var jsonData map[string][]string
  json.Unmarshal(byteValue, &jsonData)

  urlMap := make(map[string]bool)

  urls := strings.Split(output, "\n")
  for _, url := range urls {
    for get, value := range jsonData {
      if *getFlag != "" && get != *getFlag {
        continue
      }
      for _, param := range value {
        if strings.Contains(url, param) {
          if *cutFlag {
            index := strings.Index(url, "=")
            if index != -1 {
              url = url[:index+1]
            }
            if !urlMap[url] {
              urlMap[url] = true
              fmt.Println(url)
            }
          } else {
            if !urlMap[url] {
              urlMap[url] = true
              fmt.Printf("[%s] %s\n", get, url)
            }
          }
          break
        }
      }
    }
  }
}

---

Usage (Command Line)

cat urls.txt | go run script.go [-jsonUrl=<json_url>] [-cut] [-get=<category>]

---

Options

• -jsonUrl: (optional) URL to the JSON file containing parameter patterns.
  Default: already set to a predefined gist.

• -cut: (optional) Remove query strings from matched URLs.

• -get=<category>: (optional) Filter only specific parameter group from the JSON (e.g., auth, tracking).

---

Example Commands

cat log.txt | go run script.go

Show all URLs matching any pattern from the JSON.

cat log.txt | go run script.go -cut

Show matched URLs with query strings removed.

cat log.txt | go run script.go -get=auth

Filter only URLs that contain authentication-related parameters (like token=, auth=).

---