Telegram Notifications and no-color option

README.md

@@ -73,6 +73,9 @@ github_access_tokens: # provide at least one token
   - 'token one'
   - 'token two'
 slack_webhook: '' # url to your slack webhook. Found secrets will be sent here
+telegram_config:
+  token: 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' # Bot token
+  chat_id: "-1001027884121" # chat id
 blacklisted_extensions: [] # list of extensions to ignore
 blacklisted_paths: [] # list of paths to ignore
 blacklisted_entropy_extensions: [] # additional extensions to ignore for entropy checks

core/config.go

@@ -13,12 +13,19 @@ import (
 type Config struct {
 	GitHubAccessTokens           []string          `yaml:"github_access_tokens"`
 	SlackWebhook                 string            `yaml:"slack_webhook,omitempty"`
+	Telegram                     TelegramConfig    `yaml:"telegram_config,omitempty"`
 	BlacklistedExtensions        []string          `yaml:"blacklisted_extensions"`
 	BlacklistedPaths             []string          `yaml:"blacklisted_paths"`
 	BlacklistedEntropyExtensions []string          `yaml:"blacklisted_entropy_extensions"`
 	Signatures                   []ConfigSignature `yaml:"signatures"`
 }
 
+type TelegramConfig struct {
+	Token   string `yaml:"token,omitempty"`
+	ChatID  string `yaml:"chat_id,omitempty"`
+	AdminID string `yaml:"admin_id,omitempty"`
+}
+
 type ConfigSignature struct {
 	Name     string `yaml:"name"`
 	Part     string `yaml:"part"`

core/github.go

@@ -44,7 +44,7 @@ func GetRepositories(session *Session) {
 			}
 
 			if opt.Page == 0 {
-				session.Log.Warn("Token %s[..] has %d/%d calls remaining.", client.Token[:10], resp.Rate.Remaining, resp.Rate.Limit)
+				//session.Log.Warn("Token %s[..] has %d/%d calls remaining.", client.Token[:10], resp.Rate.Remaining, resp.Rate.Limit)
 			}
 
 			newEvents := make([]*github.Event, 0, len(events))

core/log.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"log"
 	"net/http"
 	"os"
 	"sync"
@@ -31,7 +32,7 @@ var LogColors = map[int]*color.Color{
 type Logger struct {
 	sync.Mutex
 
-	debug bool
+	debug  bool
 	silent bool
 }
 
@@ -43,7 +44,7 @@ func (l *Logger) SetSilent(d bool) {
 	l.silent = d
 }
 
-func (l *Logger) Log(level int, format string, args ...interface{}) {
+func (l *Logger) Log(level int, format string, file *MatchFile, args ...interface{}) {
 	l.Lock()
 	defer l.Unlock()
 
@@ -67,31 +68,73 @@ func (l *Logger) Log(level int, format string, args ...interface{}) {
 		http.Post(session.Config.SlackWebhook, "application/json", bytes.NewBuffer(jsonValue))
 	}
 
+	if session.Config.Telegram.Token != "" && session.Config.Telegram.ChatID != "" {
+		caption := fmt.Sprintf(format+"\n", args...)
+		rcpt := session.Config.Telegram.ChatID
+		if level != IMPORTANT && session.Config.Telegram.AdminID != "" {
+			rcpt = session.Config.Telegram.AdminID
+		}
+
+		if file != nil {
+			if len(caption) > 1023 {
+				caption = caption[0:1023]
+			}
+
+			values := map[string]string{
+				"caption":    caption,
+				"chat_id":    rcpt,
+				"parse_mode": "Markdown",
+			}
+			requestURL := fmt.Sprintf("https://api.telegram.org/bot%s/sendDocument", session.Config.Telegram.Token)
+			request, err := NewfileUploadRequest(requestURL, values, "document", file)
+			if err != nil {
+				log.Fatal(err)
+			}
+			client := &http.Client{}
+
+			client.Do(request)
+		} else {
+			values := map[string]string{
+				"text":       caption,
+				"chat_id":    rcpt,
+				"parse_mode": "Markdown",
+			}
+			jsonValue, _ := json.Marshal(values)
+			requestURL := fmt.Sprintf("https://api.telegram.org/bot%s/sendMessage", session.Config.Telegram.Token)
+			http.Post(requestURL, "application/json", bytes.NewBuffer(jsonValue))
+		}
+
+	}
+
 	if level == FATAL {
 		os.Exit(1)
 	}
 }
 
 func (l *Logger) Fatal(format string, args ...interface{}) {
-	l.Log(FATAL, format, args...)
+	l.Log(FATAL, format, nil, args...)
 }
 
 func (l *Logger) Error(format string, args ...interface{}) {
-	l.Log(ERROR, format, args...)
+	l.Log(ERROR, format, nil, args...)
 }
 
 func (l *Logger) Warn(format string, args ...interface{}) {
-	l.Log(WARN, format, args...)
+	l.Log(WARN, format, nil, args...)
 }
 
 func (l *Logger) Important(format string, args ...interface{}) {
-	l.Log(IMPORTANT, format, args...)
+	l.Log(IMPORTANT, format, nil, args...)
+}
+
+func (l *Logger) ImportantFile(format string, file *MatchFile, args ...interface{}) {
+	l.Log(IMPORTANT, format, file, args...)
 }
 
 func (l *Logger) Info(format string, args ...interface{}) {
-	l.Log(INFO, format, args...)
+	l.Log(INFO, format, nil, args...)
 }
 
 func (l *Logger) Debug(format string, args ...interface{}) {
-	l.Log(DEBUG, format, args...)
+	l.Log(DEBUG, format, nil, args...)
 }

core/match.go

@@ -17,7 +17,15 @@ type MatchFile struct {
 func NewMatchFile(path string) MatchFile {
 	_, filename := filepath.Split(path)
 	extension := filepath.Ext(path)
-	contents, _ := ioutil.ReadFile(path)
+	contents, err := ioutil.ReadFile(path)
+	if err != nil {
+		return MatchFile{
+			Path:      path,
+			Filename:  filename,
+			Extension: extension,
+			Contents:  []byte{},
+		}
+	}
 
 	return MatchFile{
 		Path:      path,
@@ -59,19 +67,3 @@ func (match MatchFile) CanCheckEntropy() bool {
 
 	return true
 }
-
-func GetMatchingFiles(dir string) []MatchFile {
-	fileList := make([]MatchFile, 0)
-	maxFileSize := *session.Options.MaximumFileSize * 1024
-
-	filepath.Walk(dir, func(path string, f os.FileInfo, err error) error {
-		if err != nil || f.IsDir() || uint(f.Size()) > maxFileSize || IsSkippableFile(path) {
-			return nil
-		}
-
-		fileList = append(fileList, NewMatchFile(path))
-		return nil
-	})
-
-	return fileList
-}

core/options.go

@@ -20,6 +20,7 @@ type Options struct {
 	TempDirectory          *string
 	CsvPath                *string
 	SearchQuery            *string
+	NoColor                *bool
 }
 
 func ParseOptions() (*Options, error) {
@@ -37,6 +38,7 @@ func ParseOptions() (*Options, error) {
 		TempDirectory:          flag.String("temp-directory", filepath.Join(os.TempDir(), Name), "Directory to process and store repositories/matches"),
 		CsvPath:                flag.String("csv-path", "", "CSV file path to log found secrets to. Leave blank to disable"),
 		SearchQuery:            flag.String("search-query", "", "Specify a search string to ignore signatures and filter on files containing this string (regex compatible)"),
+		NoColor:                flag.Bool("no-color", false, "Disable color output"),
 	}
 
 	flag.Parse()

core/util.go

@@ -1,9 +1,12 @@
 package core
 
 import (
+	"bytes"
 	"crypto/sha1"
 	"encoding/hex"
 	"math"
+	"mime/multipart"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
@@ -69,3 +72,29 @@ func GetEntropy(data string) (entropy float64) {
 
 	return entropy
 }
+
+func NewfileUploadRequest(uri string, params map[string]string, paramName string, file *MatchFile) (*http.Request, error) {
+	body := new(bytes.Buffer)
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile(paramName, file.Filename)
+	if err != nil {
+		return nil, err
+	}
+
+	part.Write(file.Contents)
+
+	for key, val := range params {
+		_ = writer.WriteField(key, val)
+	}
+	err = writer.Close()
+	if err != nil {
+		return nil, err
+	}
+	request, err := http.NewRequest("POST", uri, body)
+	if err != nil {
+		return nil, err
+	}
+	request.Header.Add("Content-Type", writer.FormDataContentType())
+	return request, nil
+
+}

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/fatih/color v1.7.0
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/go-querystring v1.0.0 // indirect
+	github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334
 	github.com/mattn/go-colorable v0.1.2 // indirect
 	github.com/mattn/go-isatty v0.0.9 // indirect
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45

go.sum

@@ -20,6 +20,8 @@ github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4r
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 h1:VHgatEHNcBFEB7inlalqfNqw65aNkM1lGX2yt3NmbS8=
+github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=

main.go

@@ -4,11 +4,15 @@ import (
 	"bufio"
 	"bytes"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 
 	"github.com/eth0izzle/shhgit/core"
 	"github.com/fatih/color"
+
+	//_ "net/http/pprof"
+	"github.com/iancoleman/strcase"
 )
 
 var session = core.GetSession()
@@ -24,7 +28,7 @@ func ProcessRepositories() {
 				repo, err := core.GetRepository(session, repositoryId)
 
 				if err != nil {
-					session.Log.Warn("Failed to retrieve repository %d: %s", repositoryId, err)
+					//session.Log.Warn("Failed to retrieve repository %d: %s", repositoryId, err)
 					continue
 				}
 
@@ -54,8 +58,8 @@ func ProcessGists() {
 
 func processRepositoryOrGist(url string) {
 	var (
-		matches    []string
-		matchedAny bool = false
+		matches []string
+		//matchedAny bool = false
 	)
 
 	dir := core.GetTempDir(core.GetHash(url))
@@ -69,7 +73,16 @@ func processRepositoryOrGist(url string) {
 
 	session.Log.Debug("[%s] Cloning in to %s", url, strings.Replace(dir, *session.Options.TempDirectory, "", -1))
 
-	for _, file := range core.GetMatchingFiles(dir) {
+	maxFileSize := *session.Options.MaximumFileSize * 1024
+	defer os.RemoveAll(dir)
+
+	filepath.Walk(dir, func(path string, f os.FileInfo, err error) error {
+		if err != nil || f.IsDir() || uint(f.Size()) > maxFileSize || core.IsSkippableFile(path) {
+			return nil
+		}
+
+		file := core.NewMatchFile(path)
+		defer os.Remove(file.Path)
 		relativeFileName := strings.Replace(file.Path, *session.Options.TempDirectory, "", -1)
 
 		if *session.Options.SearchQuery != "" {
@@ -81,24 +94,23 @@ func processRepositoryOrGist(url string) {
 			if matches != nil {
 				count := len(matches)
 				m := strings.Join(matches, ", ")
-				session.Log.Important("[%s] %d %s for %s in file %s: %s", url, count, core.Pluralize(count, "match", "matches"), color.GreenString("Search Query"), relativeFileName, color.YellowString(m))
+				session.Log.ImportantFile("[%s] %d %s for %s in file %s: %s", &file, url, count, core.Pluralize(count, "match", "matches"), color.GreenString("Search Query"), relativeFileName, color.YellowString(m))
 				session.WriteToCsv([]string{url, "Search Query", relativeFileName, m})
 			}
 		} else {
 			for _, signature := range session.Signatures {
 				if matched, part := signature.Match(file); matched {
-					matchedAny = true
 
 					if part == core.PartContents {
 						if matches = signature.GetContentsMatches(file); matches != nil {
 							count := len(matches)
 							m := strings.Join(matches, ", ")
-							session.Log.Important("[%s] %d %s for %s in file %s: %s", url, count, core.Pluralize(count, "match", "matches"), color.GreenString(signature.Name()), relativeFileName, color.YellowString(m))
+							session.Log.ImportantFile("#%s\n\n%s\n%d %s for %s: `%s`", &file, strcase.ToCamel(signature.Name()), url, count, core.Pluralize(count, "match", "matches"), color.GreenString(signature.Name()), color.YellowString(m))
 							session.WriteToCsv([]string{url, signature.Name(), relativeFileName, m})
 						}
 					} else {
 						if *session.Options.PathChecks {
-							session.Log.Important("[%s] Matching file %s for %s", url, color.YellowString(relativeFileName), color.GreenString(signature.Name()))
+							session.Log.ImportantFile("#%s\n\n%s\n`%s`", &file, strcase.ToCamel(signature.Name()), url, color.GreenString(signature.Name()))
 							session.WriteToCsv([]string{url, signature.Name(), relativeFileName, ""})
 						}
 
@@ -112,7 +124,7 @@ func processRepositoryOrGist(url string) {
 									entropy := core.GetEntropy(scanner.Text())
 
 									if entropy >= *session.Options.EntropyThreshold {
-										session.Log.Important("[%s] Potential secret in %s = %s", url, color.YellowString(relativeFileName), color.GreenString(scanner.Text()))
+										session.Log.ImportantFile("#PotentialSecret\n\n%s\n`%s`", &file, url, color.GreenString(scanner.Text()))
 										session.WriteToCsv([]string{url, signature.Name(), relativeFileName, scanner.Text()})
 									}
 								}
@@ -123,14 +135,8 @@ func processRepositoryOrGist(url string) {
 			}
 		}
 
-		if !matchedAny {
-			os.Remove(file.Path)
-		}
-	}
-
-	if !matchedAny {
-		os.RemoveAll(dir)
-	}
+		return nil
+	})
 }
 
 func main() {
@@ -140,6 +146,10 @@ func main() {
 		session.Log.Important("Search Query '%s' given. Only returning matching results.", *session.Options.SearchQuery)
 	}
 
+	if *session.Options.NoColor == true {
+		color.NoColor = true
+	}
+
 	go core.GetRepositories(session)
 	go ProcessRepositories()
 
@@ -149,5 +159,6 @@ func main() {
 	}
 
 	session.Log.Info("Press Ctrl+C to stop and exit.\n")
+	//log.Fatal(http.ListenAndServe(":8080", nil))
 	select {}
 }