feat(installer)!: allow ec2 instance profile by default (#259)

BREAKING CHANGE: By default, the worker agent now allows an EC2 instance
profile. Users that do not want to allow the EC2 instance profile must
configure the "allow_ec2_instance_profile" option to false. This can be
done via the config file, environment variable, or command-line
arguments. There is also a new installer option
"--disallow-instance-profile" that can be used to set the config file
option to false at install time.

Signed-off-by: Jericho Tolentino <68654047+jericht@users.noreply.github.com>

src/deadline_worker_agent/installer/__init__.py

@@ -87,6 +87,7 @@ def install() -> None:
             allow_shutdown=args.allow_shutdown,
             parser=arg_parser,
             grant_required_access=args.grant_required_access,
+            allow_ec2_instance_profile=not args.disallow_instance_profile,
         )
         if args.user:
             installer_args.update(user_name=args.user)
@@ -127,6 +128,8 @@ def install() -> None:
             cmd.append("--no-install-service")
         if args.telemetry_opt_out:
             cmd.append("--telemetry-opt-out")
+        if args.disallow_instance_profile:
+            cmd.append("--disallow-instance-profile")
 
         try:
             run(
@@ -153,6 +156,7 @@ class ParsedCommandLineArguments(Namespace):
     telemetry_opt_out: bool
     vfs_install_path: str
     grant_required_access: bool
+    disallow_instance_profile: bool
 
 
 def get_argument_parser() -> ArgumentParser:  # pragma: no cover
@@ -232,6 +236,17 @@ def get_argument_parser() -> ArgumentParser:  # pragma: no cover
         "--vfs-install-path",
         help="Absolute path for the install location of the deadline vfs.",
     )
+    parser.add_argument(
+        "--disallow-instance-profile",
+        help=(
+            "Disallow running the worker agent with an EC2 instance profile. When this is provided, the worker "
+            "agent makes requests to the EC2 instance meta-data service (IMDS) to check for an instance profile. "
+            "If an instance profile is detected, the worker agent will stop and exit. When this is not provided, "
+            "the worker agent no longer performs these checks, allowing it to run with an EC2 instance profile."
+        ),
+        action="store_true",
+        default=False,
+    )
 
     if sys.platform == "win32":
         parser.add_argument(

src/deadline_worker_agent/installer/install.sh

@@ -41,6 +41,7 @@ scripts_path="unset"
 worker_agent_program="deadline-worker-agent"
 client_library_program="deadline"
 allow_shutdown="no"
+disallow_instance_profile="no"
 no_install_service="no"
 start_service="no"
 telemetry_opt_out="no"
@@ -90,6 +91,11 @@ usage()
     echo "    --vfs-install-path VFS_INSTALL_PATH"
     echo "        An optional, absolute path to the directory that the Deadline Virtual File System (VFS) is"
     echo "        installed."
+    echo "    --disallow-instance-profile"
+    echo "        Disallow running the worker agent with an EC2 instance profile. When this is provided, the worker "
+    echo "        agent makes requests to the EC2 instance meta-data service (IMDS) to check for an instance profile. "
+    echo "        If an instance profile is detected, the worker agent will stop and exit. When this is not provided, "
+    echo "        the worker agent no longer performs these checks, allowing it to run with an EC2 instance profile."
 
     exit 2
 }
@@ -115,7 +121,7 @@ validate_deadline_id() {
 }
 
 # Validate arguments
-PARSED_ARGUMENTS=$(getopt -n install.sh --longoptions farm-id:,fleet-id:,region:,user:,group:,scripts-path:,vfs-install-path:,start,allow-shutdown,no-install-service,telemetry-opt-out -- "y" "$@")
+PARSED_ARGUMENTS=$(getopt -n install.sh --longoptions farm-id:,fleet-id:,region:,user:,group:,scripts-path:,vfs-install-path:,start,allow-shutdown,no-install-service,telemetry-opt-out,disallow-instance-profile -- "y" "$@")
 VALID_ARGUMENTS=$?
 if [ "${VALID_ARGUMENTS}" != "0" ]; then
     usage
@@ -128,18 +134,19 @@ eval set -- "$PARSED_ARGUMENTS"
 while :
 do
     case "${1}" in
-    --farm-id)               farm_id="$2"               ; shift 2 ;;
-    --fleet-id)              fleet_id="$2"              ; shift 2 ;;
-    --region)                region="$2"                ; shift 2 ;;
-    --user)                  wa_user="$2"               ; shift 2 ;;
-    --group)                 job_group="$2"             ; shift 2 ;;
-    --scripts-path)          scripts_path="$2"          ; shift 2 ;;
-    --vfs-install-path)      vfs_install_path="$2"      ; shift 2 ;;
-    --allow-shutdown)        allow_shutdown="yes"       ; shift   ;;
-    --no-install-service)    no_install_service="yes"   ; shift   ;;
-    --telemetry-opt-out)     telemetry_opt_out="yes"    ; shift   ;;
-    --start)                 start_service="yes"        ; shift   ;;
-    -y)                      confirm="$1"               ; shift   ;;
+    --farm-id)                      farm_id="$2"                    ; shift 2 ;;
+    --fleet-id)                     fleet_id="$2"                   ; shift 2 ;;
+    --region)                       region="$2"                     ; shift 2 ;;
+    --user)                         wa_user="$2"                    ; shift 2 ;;
+    --group)                        job_group="$2"                  ; shift 2 ;;
+    --scripts-path)                 scripts_path="$2"               ; shift 2 ;;
+    --vfs-install-path)             vfs_install_path="$2"           ; shift 2 ;;
+    --allow-shutdown)               allow_shutdown="yes"            ; shift   ;;
+    --disallow-instance-profile)    disallow_instance_profile="yes" ; shift   ;;
+    --no-install-service)           no_install_service="yes"        ; shift   ;;
+    --telemetry-opt-out)            telemetry_opt_out="yes"         ; shift   ;;
+    --start)                        start_service="yes"             ; shift   ;;
+    -y)                             confirm="$1"                    ; shift   ;;
     # -- means the end of the arguments; drop this, and break out of the while loop
     --) shift; break ;;
     # If non-valid options were passed, then getopt should have reported an error,
@@ -256,6 +263,7 @@ echo "Allow worker agent shutdown: ${allow_shutdown}"
 echo "Start systemd service: ${start_service}"
 echo "Telemetry opt-out: ${telemetry_opt_out}"
 echo "VFS install path: ${vfs_install_path}"
+echo "Disallow EC2 instance profile: ${disallow_instance_profile}"
 
 # Confirmation prompt
 if [ -z "$confirm" ]; then
@@ -378,14 +386,21 @@ if [[ "${allow_shutdown}" == "yes" ]]; then
 else
    shutdown_on_stop="false"
 fi
+if [[ "${disallow_instance_profile}" == "yes" ]]; then
+   allow_ec2_instance_profile="false"
+else
+   allow_ec2_instance_profile="true"
+fi
 
 echo "Configuring farm and fleet"
 echo "Configuring shutdown on stop"
+echo "Configuring allow ec2 instance profile"
 sed -E                                                          \
     --in-place=.bak                                             \
     -e "s,^# farm_id\s*=\s*\"REPLACE-WITH-WORKER-FARM-ID\"$,farm_id = \"${farm_id}\",g"    \
     -e "s,^# fleet_id\s*=\s*\"REPLACE-WITH-WORKER-FLEET-ID\"$,fleet_id = \"${fleet_id}\",g" \
     -e "s,^[#]*\s*shutdown_on_stop\s*=\s*\w+$,shutdown_on_stop = ${shutdown_on_stop},g"    \
+    -e "s,^[#]*\s*allow_ec2_instance_profile\s*=\s*\w+$,allow_ec2_instance_profile = ${allow_ec2_instance_profile},g"    \
     /etc/amazon/deadline/worker.toml
 if ! grep "farm_id = \"${farm_id}\"" /etc/amazon/deadline/worker.toml; then
     echo "ERROR: Failed to configure farm ID in /etc/amazon/deadline/worker.toml."
@@ -395,7 +410,17 @@ if ! grep "fleet_id = \"${fleet_id}\"" /etc/amazon/deadline/worker.toml; then
     echo "ERROR: Failed to configure fleet ID in /etc/amazon/deadline/worker.toml."
     exit 1
 fi
+if ! grep "shutdown_on_stop = ${shutdown_on_stop}" /etc/amazon/deadline/worker.toml; then
+    echo "ERROR: Failed to configure shutdown on stop in /etc/amazon/deadline/worker.toml."
+    exit 1
+fi
+if ! grep "allow_ec2_instance_profile = ${allow_ec2_instance_profile}" /etc/amazon/deadline/worker.toml; then
+    echo "ERROR: Failed to configure allow ec2 instance profile in /etc/amazon/deadline/worker.toml."
+    exit 1
+fi
 echo "Done configuring farm and fleet"
+echo "Done configuring shutdown on stop"
+echo "Done configuring allow ec2 instance profile"
 
 if ! [[ "${no_install_service}" == "yes" ]]; then
     # Set up the service

src/deadline_worker_agent/installer/win_installer.py

@@ -307,6 +307,7 @@ def update_config_file(
     farm_id: str,
     fleet_id: str,
     shutdown_on_stop: Optional[bool] = None,
+    allow_ec2_instance_profile: Optional[bool] = None,
 ) -> None:
     """
     Updates the worker configuration file, creating it from the example if it does not exist.
@@ -387,6 +388,24 @@ def update_config_file(
             )
         else:
             updated_keys.append("shutdown_on_stop")
+    if allow_ec2_instance_profile is not None:
+        allow_ec2_instance_profile_toml = str(allow_ec2_instance_profile).lower()
+        content = re.sub(
+            r"^#*\s*allow_ec2_instance_profile\s*=\s*\w+$",
+            f"allow_ec2_instance_profile = {allow_ec2_instance_profile_toml}",
+            content,
+            flags=re.MULTILINE,
+        )
+        if not re.search(
+            rf"^allow_ec2_instance_profile = {re.escape(allow_ec2_instance_profile_toml)}$",
+            content,
+            flags=re.MULTILINE,
+        ):
+            raise InstallerFailedException(
+                f"Failed to configure allow_ec2_instance_profile in {worker_config_file}"
+            )
+        else:
+            updated_keys.append("allow_ec2_instance_profile")
 
     # Write the updated content back to the worker configuration file
     with open(worker_config_file, "w") as file:
@@ -749,6 +768,7 @@ def start_windows_installer(
     confirm: bool = False,
     telemetry_opt_out: bool = False,
     grant_required_access: bool = False,
+    allow_ec2_instance_profile: bool = True,
 ):
     logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
 
@@ -798,7 +818,8 @@ def print_helping_info_and_exit():
         f"Allow worker agent shutdown: {allow_shutdown}\n"
         f"Install Windows service: {install_service}\n"
         f"Start service: {start_service}\n"
-        f"Telemetry opt-out: {telemetry_opt_out}"
+        f"Telemetry opt-out: {telemetry_opt_out}\n"
+        f"Disallow EC2 instance profile: {not allow_ec2_instance_profile}"
     )
     print()
 
@@ -896,6 +917,7 @@ def print_helping_info_and_exit():
         # This always sets shutdown_on_stop even if the user did not provide
         # any "shutdown" option to be consistent with POSIX installer
         shutdown_on_stop=allow_shutdown,
+        allow_ec2_instance_profile=allow_ec2_instance_profile,
     )
 
     if telemetry_opt_out:

src/deadline_worker_agent/startup/cli_args.py

@@ -19,7 +19,7 @@ class ParsedCommandLineArguments(Namespace):
     no_impersonation: bool | None = None
     run_jobs_as_agent_user: bool | None = None
     posix_job_user: str | None = None
-    allow_instance_profile: bool | None = None
+    disallow_instance_profile: bool | None = None
     logs_dir: Path | None = None
     local_session_logs: bool | None = None
     persistence_dir: Path | None = None
@@ -118,14 +118,23 @@ def get_argument_parser() -> ArgumentParser:
         action="store_true",
         dest="no_allow_instance_profile",
     )
+    # TODO: This is deprecated. Remove this eventually
     parser.add_argument(
         "--allow-instance-profile",
-        help="Turns off validation that the host EC2 instance profile is disassociated before starting",
+        help="DEPRECATED. This does nothing",
         action="store_const",
         const=True,
         dest="allow_instance_profile",
         default=None,
     )
+    parser.add_argument(
+        "--disallow-instance-profile",
+        help="Turns on validation that the host EC2 instance profile is disassociated before starting",
+        action="store_const",
+        const=True,
+        dest="disallow_instance_profile",
+        default=None,
+    )
     parser.add_argument(
         "--host-metrics-logging-interval-seconds",
         help="The interval between host metrics log messages. Default is 60.",

src/deadline_worker_agent/startup/config.py

@@ -124,8 +124,10 @@ def __init__(
             settings_kwargs["run_jobs_as_agent_user"] = parsed_cli_args.no_impersonation
         if parsed_cli_args.posix_job_user is not None:
             settings_kwargs["posix_job_user"] = parsed_cli_args.posix_job_user
-        if parsed_cli_args.allow_instance_profile is not None:
-            settings_kwargs["allow_instance_profile"] = parsed_cli_args.allow_instance_profile
+        if parsed_cli_args.disallow_instance_profile is not None:
+            settings_kwargs[
+                "allow_instance_profile"
+            ] = not parsed_cli_args.disallow_instance_profile
         if parsed_cli_args.logs_dir is not None:
             settings_kwargs["worker_logs_dir"] = parsed_cli_args.logs_dir.absolute()
         if parsed_cli_args.persistence_dir is not None:

src/deadline_worker_agent/startup/settings.py

@@ -57,10 +57,11 @@ class WorkerSettings(BaseSettings):
     windows_job_user_password_arn : str
         The ARN of an AWS Secrets Manager secret containing the password of the job user for Windows.
     allow_instance_profile : bool
-        If false (the default) and the worker is running on an EC2 instance with IMDS, then the
+        If false and the worker is running on an EC2 instance with IMDS, then the
         worker will wait until the instance profile is disassociated before running worker sessions.
         This will repeatedly attempt to make requests to IMDS. If the instance profile is still
-        associated after some threshold, the worker agent program will log the error and exit .
+        associated after some threshold, the worker agent program will log the error and exit.
+        Default is true.
     capabilities : deadline_worker_agent.startup.Capabilities
         A set of capabilities that will be declared when the worker starts. These capabilities
         can be used by the service to determine if the worker is eligible to run sessions for a
@@ -96,7 +97,7 @@ class WorkerSettings(BaseSettings):
     windows_job_user_password_arn: Optional[str] = Field(
         regex=r"^arn:aws:secretsmanager:[a-z0-9\-]+:\d{12}:secret\/[a-zA-Z0-9/_+=.@-]+$"
     )
-    allow_instance_profile: bool = False
+    allow_instance_profile: bool = True
     capabilities: Capabilities = Field(
         default_factory=lambda: Capabilities(amounts={}, attributes={})
     )

test/conftest.py

@@ -83,6 +83,11 @@ def grant_required_access() -> bool:
     return True
 
 
+@pytest.fixture
+def disallow_instance_profile() -> bool:
+    return True
+
+
 @pytest.fixture
 def parsed_args(
     farm_id: str,
@@ -98,6 +103,7 @@ def parsed_args(
     telemetry_opt_out: bool,
     vfs_install_path: str,
     grant_required_access: bool,
+    disallow_instance_profile: bool,
 ) -> ParsedCommandLineArguments:
     parsed_args = ParsedCommandLineArguments()
     parsed_args.farm_id = farm_id
@@ -113,6 +119,7 @@ def parsed_args(
     parsed_args.telemetry_opt_out = telemetry_opt_out
     parsed_args.vfs_install_path = vfs_install_path
     parsed_args.grant_required_access = grant_required_access
+    parsed_args.disallow_instance_profile = disallow_instance_profile
     return parsed_args
 
 

test/unit/install/test_install.py

@@ -77,6 +77,8 @@ def expected_cmd(
         expected_cmd.append("--allow-shutdown")
     if parsed_args.telemetry_opt_out:
         expected_cmd.append("--telemetry-opt-out")
+    if parsed_args.disallow_instance_profile:
+        expected_cmd.append("--disallow-instance-profile")
     return expected_cmd
 
 

test/unit/install/test_windows_installer.py

@@ -52,6 +52,7 @@ def test_start_windows_installer(
                 allow_shutdown=parsed_args.allow_shutdown,
                 telemetry_opt_out=parsed_args.telemetry_opt_out,
                 grant_required_access=parsed_args.grant_required_access,
+                allow_ec2_instance_profile=not parsed_args.disallow_instance_profile,
             )
 
 

test/unit/startup/test_bootstrap.py

@@ -119,7 +119,7 @@ def config(
         cli_args.no_shutdown = no_shutdown
         cli_args.profile = profile
         cli_args.verbose = verbose
-        cli_args.allow_instance_profile = allow_instance_profile
+        cli_args.disallow_instance_profile = not allow_instance_profile
         # Direct the logs and persistence state into a temporary directory
         cli_args.logs_dir = Path(tempdir) / "temp-logs-dir"
         cli_args.persistence_dir = Path(tempdir) / "temp-persist-dir"