JSON Output support

[jaguasch]

Instead of printing, I added an alternative of outputting the result in JSON format, if flag '--json' is specified.

The format of the JSON contains date, branch, commit, full diff and the string that matched

$ python truffleHog.py -h
usage: truffleHog.py [-h] [--json] git_url

Find secrets hidden in the depths of git.

positional arguments:
  git_url     URL for secret searching

optional arguments:
  -h, --help  show this help message and exit
  --json      Output in JSON

$ python truffleHog.py --json https://github.com/dxa4481/truffleHog.git | jq .

{
  "date": "2017-01-02 18:21:17",
  "commit": "adding python3 compatability\n",
  "string": "1234567890abcdefABCDEF",
  "branch": "master",
  "diff": "@@ -1,9 +1,8 @@\n import shutil, sys, math, string, datetime, argparse, tempfile\n from git import Repo\n \n-if sys.version_info[0] == 2:\n-    reload(sys)  \n-    sys.setdefaultencoding('utf8')\n+reload(sys)  \n+sys.setdefaultencoding('utf8')\n \n BASE64_CHARS = \"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\"\n HEX_CHARS = \"1234567890abcdefABCDEF\"\n@@ -72,9 +71,9 @@ def find_strings(git_url):\n                 diff = prev_commit.diff(curr_commit, create_patch=True)\n                 for blob in diff:\n                     #print i.a_blob.data_stream.read()\n-                    printableDiff = blob.diff.decode() \n+                    printableDiff = blob.diff\n                     foundSomething = False\n-                    lines = blob.diff.decode().split(\"\\n\")\n+                    lines = blob.diff.split(\"\\n\")\n                     for line in lines:\n                         for word in line.split():\n                             base64_strings = get_strings_of_set(word, BASE64_CHARS)\n@@ -91,10 +90,10 @@ def find_strings(git_url):\n                                     printableDiff = printableDiff.replace(string, bcolors.WARNING + string + bcolors.ENDC)\n                     if foundSomething:\n                         commit_time =  datetime.datetime.fromtimestamp(prev_commit.committed_date).strftime('%Y-%m-%d %H:%M:%S')\n-                        print(bcolors.OKGREEN + \"Date: \" + commit_time + bcolors.ENDC)\n-                        print(bcolors.OKGREEN + \"Branch: \" + branch_name + bcolors.ENDC)\n-                        print(bcolors.OKGREEN + \"Commit: \" + prev_commit.message + bcolors.ENDC)\n-                        print(printableDiff)\n+                        print bcolors.OKGREEN + \"Date: \" + commit_time + bcolors.ENDC\n+                        print bcolors.OKGREEN + \"Branch: \" + branch_name + bcolors.ENDC\n+                        print bcolors.OKGREEN + \"Commit: \" + prev_commit.message + bcolors.ENDC\n+                        print printableDiff\n                     \n             prev_commit = curr_commit\n     shutil.rmtree(project_path)\n"
}
{
  "date": "2017-01-02 17:44:30",
  "commit": "adding attribution\n",
  "string": "1234567890abcdefABCDEF",
  "branch": "master",
  "diff": "@@ -8,9 +8,6 @@ BASE64_CHARS = \"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/\n HEX_CHARS = \"1234567890abcdefABCDEF\"\n \n def shannon_entropy(data, iterator):\n-    \"\"\"\n-    Borrowed from http://blog.dkbza.org/2007/05/scanning-data-for-entropy-anomalies.html\n-    \"\"\"\n     if not data:\n         return 0\n     entropy = 0\n"
}
{
  "date": "2017-01-02 01:53:49",
  "commit": "fixing unicode commit message problem\n",
  "string": "1234567890abcdefABCDEF",
  "branch": "master",
  "diff": "@@ -1,10 +1,7 @@\n-import shutil, sys, os, math, string, datetime, argparse\n+import shutil, os, math, string, datetime, argparse\n from uuid import uuid4\n from git import Repo\n \n-reload(sys)  \n-sys.setdefaultencoding('utf8')\n-\n BASE64_CHARS = \"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\"\n HEX_CHARS = \"1234567890abcdefABCDEF\"\n \n@@ -93,6 +90,7 @@ def find_strings(git_url):\n                         print bcolors.OKGREEN + \"Branch: \" + branch_name + bcolors.ENDC\n                         print bcolors.OKGREEN + \"Commit: \" + prev_commit.message + bcolors.ENDC\n                         print printableDiff\n+\n                     \n             prev_commit = curr_commit\n     shutil.rmtree(project_path)\n"
}
{
  "date": "2016-12-31 05:21:09",
  "commit": "Nothing to see here\n",
  "string": "+iZNOIPugKnpjh6D6tyNmeGmU7lyPfhIIqGeKYOoyFN9WUB9ZaDPQmv3sPi3g5wFY3UxmhowWYD1GYVq7E54xJYMvrLH6yJt8UlN4",
  "branch": "master",
  "diff": "@@ -1 +1 @@\n-nothing\n+iZNOIPugKnpjh6D6tyNmeGmU7lyPfhIIqGeKYOoyFN9WUB9ZaDPQmv3sPi3g5wFY3UxmhowWYD1GYVq7E54xJYMvrLH6yJt8UlN4.\n"
}
{
  "date": "2016-12-31 05:20:50",
  "commit": "OH no a secret\n",
  "string": "iZNOIPugKnpjh6D6tyNmeGmU7lyPfhIIqGeKYOoyFN9WUB9ZaDPQmv3sPi3g5wFY3UxmhowWYD1GYVq7E54xJYMvrLH6yJt8UlN4",
  "branch": "master",
  "diff": "@@ -1 +1 @@\n-iZNOIPugKnpjh6D6tyNmeGmU7lyPfhIIqGeKYOoyFN9WUB9ZaDPQmv3sPi3g5wFY3UxmhowWYD1GYVq7E54xJYMvrLH6yJt8UlN4.\n+.\n"
}
{
  "date": "2016-12-31 05:19:51",
  "commit": "removing secret\n",
  "string": "+iZNOIPugKnpjh6D6tyNmeGmU7lyPfhIIqGeKYOoyFN9WUB9ZaDPQmv3sPi3g5wFY3UxmhowWYD1GYVq7E54xJYMvrLH6yJt8UlN4",
  "branch": "master",
  "diff": "@@ -0,0 +1 @@\n+iZNOIPugKnpjh6D6tyNmeGmU7lyPfhIIqGeKYOoyFN9WUB9ZaDPQmv3sPi3g5wFY3UxmhowWYD1GYVq7E54xJYMvrLH6yJt8UlN4\n"
}
{
  "date": "2016-12-31 05:19:39",
  "commit": "Oh no a secret file\n",
  "string": "iZNOIPugKnpjh6D6tyNmeGmU7lyPfhIIqGeKYOoyFN9WUB9ZaDPQmv3sPi3g5wFY3UxmhowWYD1GYVq7E54xJYMvrLH6yJt8UlN4",
  "branch": "master",
  "diff": "@@ -1 +0,0 @@\n-iZNOIPugKnpjh6D6tyNmeGmU7lyPfhIIqGeKYOoyFN9WUB9ZaDPQmv3sPi3g5wFY3UxmhowWYD1GYVq7E54xJYMvrLH6yJt8UlN4\n"
}
{
  "date": "2016-12-31 05:10:07",
  "commit": "adding initial files\n",
  "string": "1234567890abcdefABCDEF",
  "branch": "master",
  "diff": "@@ -1,101 +0,0 @@\n-import os, math, string\n-import argparse\n-from uuid import uuid4\n-from git import Repo\n-\n-BASE64_CHARS = \"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\"\n-HEX_CHARS = \"1234567890abcdefABCDEF\"\n-\n-def shannon_entropy(data, iterator):\n-    if not data:\n-        return 0\n-    entropy = 0\n-    for x in (ord(c) for c in iterator):\n-        p_x = float(data.count(chr(x)))/len(data)\n-        if p_x > 0:\n-            entropy += - p_x*math.log(p_x, 2)\n-    return entropy\n-\n-\n-def get_strings_of_set(word, char_set, threshold=20):\n-    count = 0\n-    letters = \"\"\n-    strings = []\n-    for char in word:\n-        if char in char_set:\n-            letters += char\n-            count += 1\n-        else:\n-            if count > 20:\n-                strings.append(letters)\n-            letters = \"\"\n-            count = 0\n-    if count > threshold:\n-        strings.append(letters)\n-    return strings\n-\n-class bcolors:\n-    HEADER = '\\033[95m'\n-    OKBLUE = '\\033[94m'\n-    OKGREEN = '\\033[92m'\n-    WARNING = '\\033[93m'\n-    FAIL = '\\033[91m'\n-    ENDC = '\\033[0m'\n-    BOLD = '\\033[1m'\n-    UNDERLINE = '\\033[4m'\n-\n-def find_strings(git_url):\n-    new_project = str(uuid4())\n-    project_path = os.path.join(\"temp\", new_project)\n-\n-    Repo.clone_from(git_url, project_path)\n-\n-    repo = Repo(project_path)\n-\n-\n-    for i in repo.remotes.origin.fetch():\n-        branch_name = str(i).split('/')[1]\n-        try:\n-            repo.git.checkout(i, b=branch_name)\n-        except:\n-            pass\n-     \n-        prev_commit = None\n-        for curr_commit in repo.iter_commits():\n-            if not prev_commit:\n-                pass\n-            else:\n-                diff = prev_commit.diff(curr_commit, create_patch=True)\n-                for i in diff:\n-                    #print i.a_blob.data_stream.read()\n-                    printableDiff = i.diff\n-                    foundSomething = False\n-                    lines = i.diff.split(\"\\n\")\n-                    for line in lines:\n-                        for word in line.split():\n-                            base64_strings = get_strings_of_set(word, BASE64_CHARS)\n-                            hex_strings = get_strings_of_set(word, HEX_CHARS)\n-                            for string in base64_strings:\n-                                b64Entropy = shannon_entropy(string, BASE64_CHARS)\n-                                if b64Entropy > 4.5:\n-                                    foundSomething = True\n-                                    printableDiff = printableDiff.replace(string, bcolors.WARNING + string + bcolors.ENDC)\n-                            for string in hex_strings:\n-                                hexEntropy = shannon_entropy(string, HEX_CHARS)\n-                                if hexEntropy > 3:\n-                                    foundSomething = True\n-                                    printableDiff = printableDiff.replace(string, bcolors.WARNING + string + bcolors.ENDC)\n-                    if foundSomething:\n-                        print printableDiff\n-\n-                    \n-            prev_commit = curr_commit\n-\n-if __name__ == \"__main__\":\n-    parser = argparse.ArgumentParser(description='Find secrets hidden in the depths of git.')\n-    parser.add_argument('git_url', type=str, help='URL for secret searching')\n-\n-\n-    args = parser.parse_args()\n-    find_strings(args.git_url)\n-\n"
}