Build Ruby 3.1

Author: adfoster-r7

.github/workflows/verify.yml

@@ -28,8 +28,7 @@ jobs:
   # The job checkout structure is:
   #  .
   #  ├── metasploit-omnibus
-  #  └── metasploit-framework (Only if ARM/Windows builds)
-  # For windows we additionally move metasploit-framework into the omnibus local cache
+  #  └── metasploit-framework (Only if ARM or Windows builds)
   #
   docker_arm:
     runs-on: ${{ matrix.os }}
@@ -264,12 +263,27 @@ jobs:
       fail-fast: false
       matrix:
         os:
-          - windows-2019
+          - windows-2022
         ruby:
           - 3.0.6
 
     name: ${{ matrix.os }}
     steps:
+      # https://github.com/actions/runner-images/issues/5143
+      # https://github.com/actions/runner-images/issues/9701
+      - name: Install visual studio components
+        run: |
+          Set-Location "C:\Program Files (x86)\Microsoft Visual Studio\Installer\"
+          $InstallPath = "C:\Program Files\Microsoft Visual Studio\2022\Enterprise"
+          $componentsToRemove= @(
+            "Microsoft.VisualStudio.Component.VC.Redist.MSM"
+          )
+          [string]$workloadArgs = $componentsToRemove | ForEach-Object {" --add " +  $_}
+          $Arguments = ('/c', "vs_installer.exe", 'modify', '--installPath', "`"$InstallPath`"",$workloadArgs, '--quiet', '--norestart', '--nocache')
+          # should be run twice
+          $process = Start-Process -FilePath cmd.exe -ArgumentList $Arguments -Wait -PassThru -WindowStyle Hidden
+          $process = Start-Process -FilePath cmd.exe -ArgumentList $Arguments -Wait -PassThru -WindowStyle Hidden
+
       - name: Checkout omnibus
         uses: actions/checkout@v4
         with:
@@ -289,6 +303,20 @@ jobs:
           # https://github.com/ruby/setup-ruby/tree/d2b39ad0b52eca07d23f3aa14fdf2a3fcc1f411c#windows
           bundler: 2.2.33
 
+      # We need to create the pcaprub gem ourselves until 0.13.2 is released (https://github.com/pcaprub/pcaprub/issues/67)
+      - name: Checkout pcaprub
+        uses: actions/checkout@v4
+        with:
+          repository: pcaprub/pcaprub
+          path: pcaprub
+          ref: '5440ca93dafd15e7d3bb009fc1bb9a15e80d03f9'
+
+      - name: Create pcaprub gem
+        run: |
+          cd pcaprub
+          bundle
+          rake gem
+
       # Checkout framework
       - name: Checkout metasploit-framework code
         uses: actions/checkout@v4
@@ -300,12 +328,16 @@ jobs:
         run: |
           cd metasploit-omnibus
           xz -d local/cache/*.xz
+          ls local/cache
 
       - name: Run omnibus
+        shell: cmd
+        env:
+          MSYSTEM: MINGW64
         run: |
-          # mkdir -p metasploit-omnibus/local/cache/git_cache/c
-          # mv metasploit-framework/ metasploit-omnibus/local/cache/git_cache/c
-          # dir metasploit-omnibus/local/cache/git_cache/c
-          # dir metasploit-omnibus/local/cache/git_cache/c/metasploit-framework
           cd metasploit-omnibus
-          make
+          make dependencies
+
+          rem Don't run the main build itslef under `make`, as the process will be spawned under msys2
+          rem and the ridk.cmd Ruby installer will forcibly kill the msys2 process before attempting to install ruby
+          ruby bin/omnibus build metasploit-framework

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "local/cache"]
 	path = local/cache
-	url = https://github.com/rapid7/metasploit-omnibus-cache.git
+	url = https://github.com/adfoster-r7/metasploit-omnibus-cache.git
+	branch = prepare-for-ruby-3.1-update

Gemfile

@@ -2,7 +2,7 @@ source 'https://rubygems.org'
 
 # Install omnibus
 # gem 'omnibus', '~> 4.0'
-gem 'omnibus', git: 'https://github.com/rapid7/omnibus', branch: 'r7_8.2.4_custom'
+gem 'omnibus', git: 'https://github.com/rapid7/omnibus', branch: 'r7_9.0.23_custom'
 
 # Use Chef's software definitions. It is recommended that you write your own
 # software definitions, but you can clone/fork Chef's to get you started.

Gemfile.lock

@@ -1,124 +1,128 @@
 GIT
   remote: https://github.com/rapid7/omnibus
-  revision: 5b238e62088797782f6b7ca1ddda3a0e7128c224
-  branch: r7_8.2.4_custom
+  revision: c6dc312d985e85fb35b8e9f8e49b607abd91665c
+  branch: r7_9.0.23_custom
   specs:
-    omnibus (8.2.4)
-      aws-sdk-s3 (~> 1)
+    omnibus (9.0.23)
+      aws-sdk-s3 (~> 1.116.0)
       chef-cleanroom (~> 1.0)
       chef-utils (>= 15.4)
       contracts (>= 0.16.0, < 0.17.0)
       ffi-yajl (~> 2.2)
       license_scout (~> 1.0)
       mixlib-shellout (>= 2.0, < 4.0)
       mixlib-versioning
-      ohai (>= 15, < 17)
+      ohai (>= 16, < 19)
       pedump
+      rexml (~> 3.2)
       ruby-progressbar (~> 1.7)
       thor (>= 0.18, < 2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.0)
-      public_suffix (>= 2.0.2, < 5.0)
+    addressable (2.8.6)
+      public_suffix (>= 2.0.2, < 6.0)
     awesome_print (1.9.2)
-    aws-eventstream (1.2.0)
-    aws-partitions (1.522.0)
-    aws-sdk-core (3.121.5)
-      aws-eventstream (~> 1, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.520.1)
-      aws-sigv4 (~> 1.1)
-      jmespath (~> 1.0)
-    aws-sdk-kms (1.50.0)
-      aws-sdk-core (~> 3, >= 3.121.2)
+    aws-eventstream (1.3.0)
+    aws-partitions (1.916.0)
+    aws-sdk-core (3.192.1)
+      aws-eventstream (~> 1, >= 1.3.0)
+      aws-partitions (~> 1, >= 1.651.0)
+      aws-sigv4 (~> 1.8)
+      jmespath (~> 1, >= 1.6.1)
+    aws-sdk-kms (1.79.0)
+      aws-sdk-core (~> 3, >= 3.191.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.104.0)
-      aws-sdk-core (~> 3, >= 3.121.2)
+    aws-sdk-s3 (1.116.0)
+      aws-sdk-core (~> 3, >= 3.127.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.4)
-    aws-sigv4 (1.4.0)
+    aws-sigv4 (1.8.0)
       aws-eventstream (~> 1, >= 1.0.2)
     byebug (11.1.3)
     chef-cleanroom (1.0.5)
-    chef-config (16.16.13)
+    chef-config (18.4.12)
       addressable
-      chef-utils (= 16.16.13)
+      chef-utils (= 18.4.12)
       fuzzyurl
       mixlib-config (>= 2.2.12, < 4.0)
       mixlib-shellout (>= 2.0, < 4.0)
       tomlrb (~> 1.2)
-    chef-utils (16.16.13)
+    chef-utils (18.4.12)
+      concurrent-ruby
     citrus (3.0.2)
     coderay (1.1.3)
+    concurrent-ruby (1.2.3)
     contracts (0.16.1)
-    ffi (1.15.4)
-    ffi-yajl (2.4.0)
+    ffi (1.16.3)
+    ffi-yajl (2.6.0)
       libyajl2 (>= 1.2)
     fuzzyurl (0.9.0)
-    iostruct (0.0.4)
+    iostruct (0.0.5)
     ipaddress (0.8.3)
     jmespath (1.6.2)
-    json (2.6.1)
+    json (2.7.2)
     libyajl2 (2.1.0)
-    license_scout (1.2.13)
+    license_scout (1.3.7)
       ffi-yajl (~> 2.2)
       mixlib-shellout (>= 2.2, < 4.0)
       toml-rb (>= 1, < 3)
     method_source (1.0.0)
     mixlib-cli (2.1.8)
-    mixlib-config (3.0.9)
+    mixlib-config (3.0.27)
       tomlrb
     mixlib-log (3.0.9)
-    mixlib-shellout (3.2.5)
+    mixlib-shellout (3.2.7)
       chef-utils
     mixlib-versioning (1.2.12)
-    multipart-post (2.1.1)
-    net-scp (3.0.0)
-      net-ssh (>= 2.6.5, < 7.0.0)
-    net-ssh (6.1.0)
-    ohai (16.13.0)
-      chef-config (>= 12.8, < 17)
-      chef-utils (>= 16.0, < 17)
+    multipart-post (2.4.0)
+    net-scp (4.0.0)
+      net-ssh (>= 2.6.5, < 8.0.0)
+    net-ssh (7.2.3)
+    ohai (18.1.3)
+      chef-config (>= 14.12, < 19)
+      chef-utils (>= 16.0, < 19)
       ffi (~> 1.9)
       ffi-yajl (~> 2.2)
       ipaddress
       mixlib-cli (>= 1.7.0)
       mixlib-config (>= 2.0, < 4.0)
       mixlib-log (>= 2.0.1, < 4.0)
-      mixlib-shellout (>= 2.0, < 4.0)
+      mixlib-shellout (~> 3.2, >= 3.2.5)
       plist (~> 3.1)
       train-core
       wmi-lite (~> 1.0)
-    pedump (0.6.2)
+    pedump (0.6.7)
       awesome_print
       iostruct (>= 0.0.4)
       multipart-post (>= 2.0.0)
       rainbow
       zhexdump (>= 0.0.2)
-    plist (3.6.0)
-    pry (0.13.1)
+    plist (3.7.1)
+    pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    pry-byebug (3.9.0)
+    pry-byebug (3.10.1)
       byebug (~> 11.0)
-      pry (~> 0.13.0)
-    public_suffix (4.0.6)
-    rainbow (3.0.0)
-    ruby-progressbar (1.11.0)
-    thor (1.1.0)
-    toml-rb (2.1.0)
+      pry (>= 0.13, < 0.15)
+    public_suffix (5.0.5)
+    rainbow (3.1.1)
+    rexml (3.2.6)
+    ruby-progressbar (1.13.0)
+    thor (1.3.1)
+    toml-rb (2.2.0)
       citrus (~> 3.0, > 3.0)
     tomlrb (1.3.0)
-    train-core (3.8.1)
+    train-core (3.12.3)
       addressable (~> 2.5)
       ffi (!= 1.13.0)
       json (>= 1.8, < 3.0)
       mixlib-shellout (>= 2.0, < 4.0)
-      net-scp (>= 1.2, < 4.0)
-      net-ssh (>= 2.9, < 7.0)
-    wmi-lite (1.0.5)
-    zhexdump (0.0.2)
+      net-scp (>= 1.2, < 5.0)
+      net-ssh (>= 2.9, < 8.0)
+    wmi-lite (1.0.7)
+    zhexdump (0.1.0)
 
 PLATFORMS
   ruby
@@ -128,4 +132,4 @@ DEPENDENCIES
   pry-byebug
 
 BUNDLED WITH
-   2.1.4
+   2.2.33

Makefile

@@ -1,9 +1,14 @@
 .DEFAULT_GOAL := all
 
 .PHONY: all
-all: certs/ca-certificates.crt
+all: certs/ca-certificates.crt dependencies
 	# export SSL_CERT_FILE=${PWD}/certs/ca-certificates.crt
 
+	# build the metasploit-framework package
+	ruby bin/omnibus build metasploit-framework
+
+.PHONY: dependencies
+dependencies:
 	# Ensure consistent bundler versions
 	gem install bundler -v 2.2.3
 
@@ -13,9 +18,6 @@ all: certs/ca-certificates.crt
 
 	gem install win32-process -v 0.9.0
 
-	# build the metasploit-framework package
-	ruby bin/omnibus build metasploit-framework
-
 certs/ca-certificates.crt:
 	mkdir -p certs
 	curl -L -o certs/ca-certificates.crt https://curl.haxx.se/ca/cacert.pem

README.md

@@ -32,7 +32,7 @@ docker build --tag metasploit-omnibus-builder - < ./docker/ubuntu1204-x86/Docker
 Or on OSX you can use the following script to build all images following the latest Docker image naming convention:
 
 ```shell
-export BUILD_DATE=$(date "+%Y_%m"); ls ./docker | xargs -I IMAGE_NAME /bin/bash -x -c "docker build --tag rapid7/IMAGE_NAME-omnibus:$BUILD_DATE -f ./docker/IMAGE_NAME/Dockerfile ./docker/IMAGE_NAME"
+export BUILD_DATE=$(date "+%Y_%m_%d"); ls ./docker | xargs -I IMAGE_NAME /bin/bash -x -c "docker build --tag rapid7/IMAGE_NAME-omnibus:$BUILD_DATE -f ./docker/IMAGE_NAME/Dockerfile ./docker/IMAGE_NAME"
 ```
 
 You can then run a new container using the above tagged image, whilst mounting the current directory as a volume:

config/patches/ruby/ruby-fast-load_31.patch

@@ -0,0 +1,16 @@
+diff --git a/load.c b/load.c
+index a2b9da4..05ea96e 100644
+--- a/load.c
++++ b/load.c
+@@ -981,6 +981,11 @@ search_required(rb_vm_t *vm, VALUE fname, volatile VALUE *path, feature_func rb_
+        if (loading) *path = rb_filesystem_str_new_cstr(loading);
+        return 'r';
+     }
++    else if ((ft = rb_feature_p(vm, ftptr, 0, FALSE, FALSE, &loading)) == 's') {
++       if (loading) *path = rb_filesystem_str_new_cstr(loading);
++       return 's';
++    }
++
+     tmp = fname;
+     type = rb_find_file_ext(&tmp, ft == 's' ? ruby_ext : loadable_ext);
+     switch (type) {

config/software/metasploit-framework.rb

@@ -2,6 +2,9 @@
 if linux? && File.exist?("/metasploit-framework")
   # supply current version of metasploit-framework at root of filesystem
   source path: "/metasploit-framework"
+elsif windows? && File.exist?('../metasploit-framework')
+  # supply current version of metasploit-framework relative to the current directory
+  source path: "../metasploit-framework"
 else
   source git: "https://github.com/rapid7/metasploit-framework.git"
   default_version "master"
@@ -19,7 +22,7 @@
   dependency "postgresql"
 end
 
-ruby_abi_version = "3.0.0"
+ruby_abi_version = "3.1.0"
 # This depends on extra system libraries on OS X
 whitelist_file "#{install_dir}//embedded/framework/data/isight.bundle"
 
@@ -77,7 +80,9 @@
   end
 
   bundle "config set force_ruby_platform true", env: env
-  bundle "install", env: env
+  bundle_env = with_standard_compiler_flags(with_embedded_path)
+  bundle_env['MAKE'] = 'make -j4'
+  bundle "install --jobs=4", env: bundle_env
 
   if windows?
     delete "#{install_dir}/embedded/msys64"