Merge pull request #19779 from h00die/action_update_weekly

Weekly Updater Action

.github/workflows/weekly-data-and-external-tool-updater.yml

@@ -0,0 +1,96 @@
+name: Weekly Data and External Tool Updater
+
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions
+permissions:
+  actions: none
+  checks: none
+  contents: none
+  deployments: none
+  id-token: none
+  issues: none
+  discussions: none
+  packages: none
+  pages: none
+  pull-requests: write
+  repository-projects: none
+  security-events: none
+  statuses: none
+
+on:
+  schedule:
+    # Run once a week (e.g., every Monday at 01:00 UTC)
+    - cron: '0 1 * * 1'
+  workflow_dispatch: # Allows manual triggering from the Actions tab
+
+jobs:
+  update-data-files:
+    runs-on: ubuntu-latest
+
+    env:
+      BUNDLE_WITHOUT: "coverage development pcap"
+
+    strategy:
+      fail-fast: true
+      matrix:
+        ruby:
+          - '3.2'
+
+    steps:
+      - name: Install system dependencies
+        run: sudo apt-get install libpcap-dev graphviz
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '${{ matrix.ruby }}'
+          bundler-cache: true
+
+      - name: Run Ruby updater scripts
+        run: |
+          ruby tools/dev/update_wordpress_vulnerabilities.rb
+          ruby tools/dev/update_joomla_components.rb
+          ruby tools/dev/update_user_agent_strings.rb
+          ruby tools/dev/check_external_scripts.rb -u
+      - name: Remove vendor folder # prevent git from adding it
+        run: rm -rf vendor
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: Update report
+          base: master
+          branch: weekly-updates
+          committer: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
+          author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
+          title: "Weekly Data Update"
+          draft: false
+          body: |
+            This pull request was created automatically by a GitHub Action to update data files and external scripts.
+            The following tools were run:
+            - ruby tools/dev/update_wordpress_vulnerabilities.rb
+            - ruby tools/dev/update_joomla_components.rb
+            - ruby tools/dev/update_user_agent_strings.rb
+            - ruby tools/dev/check_external_scripts.rb -u
+            ## Verification
+            ### Wordpress/Joomla Files
+            - [ ] Do a sanity check, do the additions look legit?
+            - [ ] Start `msfconsole`
+            - [ ] `use modules/auxiliary/scanner/http/wordpress_scanner`
+            - [ ] **Verify** it runs
+            ### JTR Files
+            - [ ] Do a sanity check, do the additions look legit?
+            - [ ] See https://docs.metasploit.com/docs/using-metasploit/intermediate/hashes-and-password-cracking.html#example-hashes for hashes and cracking
+            ### SharpHound
+            - [ ] Start `msfconsole`
+            - [ ] get a shell on a DC or box connected to a dc
+            - [ ] `use post/windows/gather/bloodhound`
+            - [ ] `set session`
+            - [ ] `run`
+            - [ ] **Verify** it runs w/o erroring
+            - [ ] `set method disk`
+            - [ ] **Verify** it runs w/o erroring

tools/dev/update_joomla_components.rb

@@ -0,0 +1,78 @@
+#!/usr/bin/env ruby
+# -*- coding: binary -*-
+
+#
+# by h00die
+#
+
+require 'optparse'
+require 'net/http'
+require 'uri'
+optparse = OptionParser.new do |opts|
+  opts.banner = 'Usage: ruby tools/dev/update_joomla_components.rb [options]'
+  opts.separator "This program updates data/wordlists/joomla.txt which is used by modules/auxiliary/scanner/http/joomla_scanner.rb to have the most up-to-date list of vuln components"
+  opts.separator ""
+  opts.on('-h', '--help', 'Display this screen.') do
+    puts opts
+    exit
+  end
+end
+optparse.parse!
+
+# colors and puts templates from msftidy.rb
+
+class String
+  def red
+    "\e[1;31;40m#{self}\e[0m"
+  end
+
+  def yellow
+    "\e[1;33;40m#{self}\e[0m"
+  end
+
+  def green
+    "\e[1;32;40m#{self}\e[0m"
+  end
+
+  def cyan
+    "\e[1;36;40m#{self}\e[0m"
+  end
+end
+
+#
+# Display an error message, given some text
+#
+def error(txt)
+  puts "[#{'ERROR'.red}] #{cleanup_text(txt)}"
+end
+
+#
+# Display a warning message, given some text
+#
+def warning(txt)
+  puts "[#{'WARNING'.yellow}] #{cleanup_text(txt)}"
+end
+
+#
+# Display a info message, given some text
+#
+def info(txt)
+  puts "[#{'INFO'.cyan}] #{cleanup_text(txt)}"
+end
+
+uri = URI.parse('https://raw.githubusercontent.com/rezasp/joomscan/master/exploit/db/componentslist.txt')
+new_com = Net::HTTP.get(uri)
+
+old = File.read('data/wordlists/joomla.txt').split("\n")
+
+new_com.each_line do |com|
+  unless old.include?("components/#{com.strip}/")
+    old << "components/#{com.strip}/"
+    info "Adding: components/#{com.strip}/"
+  end
+end
+
+old.sort!
+File.open('data/wordlists/joomla.txt', 'w') do |file|
+  file.puts old
+end

tools/dev/update_user_agent_strings.rb

@@ -0,0 +1,112 @@
+#!/usr/bin/env ruby
+# -*- coding: binary -*-
+
+require 'optparse'
+require 'net/http'
+require 'uri'
+optparse = OptionParser.new do |opts|
+  opts.banner = 'Usage: ruby tools/dev/update_user_agent_strings.rb [options]'
+  opts.separator "This program updates lib/rex/user_agent.rb so Metasploit uses the most up-to-date User Agent strings across the framework."
+  opts.separator ""
+  opts.on('-h', '--help', 'Display this screen.') do
+    puts opts
+    exit
+  end
+end
+optparse.parse!
+
+# colors and puts templates from msftidy.rb
+
+class String
+  def red
+    "\e[1;31;40m#{self}\e[0m"
+  end
+
+  def yellow
+    "\e[1;33;40m#{self}\e[0m"
+  end
+
+  def green
+    "\e[1;32;40m#{self}\e[0m"
+  end
+
+  def cyan
+    "\e[1;36;40m#{self}\e[0m"
+  end
+end
+
+#
+# Display an error message, given some text
+#
+def error(txt)
+  puts "[#{'ERROR'.red}] #{cleanup_text(txt)}"
+end
+
+#
+# Display a warning message, given some text
+#
+def warning(txt)
+  puts "[#{'WARNING'.yellow}] #{cleanup_text(txt)}"
+end
+
+#
+# Display a info message, given some text
+#
+def info(txt)
+  puts "[#{'INFO'.cyan}] #{cleanup_text(txt)}"
+end
+
+def cleanup_text(txt)
+  # remove line breaks
+  txt = txt.gsub(/[\r\n]/, ' ')
+  # replace multiple spaces by one space
+  txt.gsub(/\s{2,}/, ' ')
+end
+
+def replace_agent_string(lines, replace_marker, url, regex)
+  valid_chars = 'a-zA-Z0-9\(\);:\.,/_ '
+  regex = regex.gsub('{VALID_CHARS}', valid_chars)
+  info "Checking: #{replace_marker}"
+
+  index = lines.index { |line| line.include?(replace_marker) }
+  raise "Couldn't find marker #{replace_marker}" if index.nil?
+
+  uri = URI(url)
+  response = Net::HTTP.get_response(uri)
+  raise "Can't retrieve #{url}" unless response.is_a?(Net::HTTPSuccess)
+
+  match = response.body.match(/#{regex}/)
+  raise "Couldn't match regex #{regex}" if match.nil?
+
+  new_string = match[1]
+
+  old_line = lines[index]
+  if old_line.include?("'#{new_string}'")
+    puts "  (Unchanged): #{new_string}"
+  else
+    new_line = old_line.gsub(/'(.*)'/, "'#{new_string}'")
+    if old_line == new_line
+      raise "  Line didn't change: #{old_line}"
+    end
+    puts "  New value is: #{new_string}"
+    lines[index] = new_line
+  end
+end
+
+chrome_url = "https://www.whatismybrowser.com/guides/the-latest-user-agent/chrome"
+edge_url = "https://www.whatismybrowser.com/guides/the-latest-user-agent/edge"
+safari_url = "https://www.whatismybrowser.com/guides/the-latest-user-agent/safari"
+firefox_url = "https://www.whatismybrowser.com/guides/the-latest-user-agent/firefox"
+
+user_agent_filename = 'lib/rex/user_agent.rb'
+lines = File.read(user_agent_filename).split("\n")
+
+replace_agent_string(lines, 'Chrome Windows', chrome_url, '<td>Chrome \\(Standard\\)</td>\s*<td>\s*<ul>\s*<li><span class="code">([{VALID_CHARS}]*Windows NT[{VALID_CHARS}]*)</span>')
+replace_agent_string(lines, 'Chrome MacOS', chrome_url, '<td>Chrome \\(Standard\\)</td>\s*<td>\s*<ul>\s*<li><span class="code">([{VALID_CHARS}]*Macintosh[{VALID_CHARS}]*)</span>')
+replace_agent_string(lines, 'Edge Windows', edge_url, '<td>Edge \\(Standard\\)</td>\s*<td>\s*<ul>\s*<li><span class="code">([{VALID_CHARS}]*Windows NT[{VALID_CHARS}]*)</span>')
+replace_agent_string(lines, 'Safari iPad', safari_url, '<td>\s*Safari on <b>Ipad</b>\s*</td>\s*<td>\s*<ul>\s*<li><span class="code">([{VALID_CHARS}]*iPad[{VALID_CHARS}]*)</span>')
+replace_agent_string(lines, 'Safari MacOS', safari_url, '<td>Safari \\(Standard\\)</td>\s*<td>\s*<ul>\s*<li><span class="code">([{VALID_CHARS}]*Macintosh[{VALID_CHARS}]*)</span>')
+replace_agent_string(lines, 'Firefox Windows', firefox_url, '<td>\s*Firefox on <b>Windows</b>\s*</td>\s*<td>\s*<ul>\s*<li><span class="code">([{VALID_CHARS}]*Windows NT[{VALID_CHARS}]*)</span>')
+replace_agent_string(lines, 'Firefox MacOS', firefox_url, '<td>\s*Firefox on <b>Macos</b>\s*</td>\s*<td>\s*<ul>\s*<li><span class="code">([{VALID_CHARS}]*Macintosh[{VALID_CHARS}]*)</span>')
+
+File.write(user_agent_filename, lines.join("\n") + "\n")

tools/dev/update_wordpress_vulnerabilities.rb

@@ -1,9 +1,6 @@
 #!/usr/bin/env ruby
 # -*- coding: binary -*-
 
-#
-# Update modules/auxiliary/scanner/http/wordpress_scanner.rb to have the most
-# up to date list of vuln components based on exploits/scanners in the framework
 #
 # by h00die
 #
@@ -12,7 +9,9 @@
 
 options = {}
 optparse = OptionParser.new do |opts|
-  opts.banner = 'Usage: update_wordpress_vulnerabilities.rb [options]'
+  opts.banner = 'Usage: ruby tools/dev/update_wordpress_vulnerabilities.rb [options]'
+  opts.separator "This program updates data/wordlists/wp-exploitable-themes.txt and wp-exploitable-plugins.txt which are used by modules/auxiliary/scanner/http/wordpress_scanner.rb to have the most up-to-date list of vuln components"
+  opts.separator ""
   opts.on('-h', '--help', 'Display this screen.') do
     puts opts
     exit