Logic improvements for tasking

conf/az.conf

@@ -2,8 +2,12 @@
 # Specify the Azure Region (for example, CanadaCentral). This is case-sensitive
 region_name = <region_name>
 
-# Resource Group for Azure
-resource_group = <resource_group>
+# Resource Groups for Azure
+# The service principal that controls the Azure resources should have READ
+# access on the virtual network where it lives, which should be in a different resource
+# group than where the sandbox resources live.
+vnet_resource_group = <resource_group>
+sandbox_resource_group = <resource_group>
 
 # Subscription ID for Azure
 subscription_id = <subscription_id>
@@ -37,6 +41,20 @@ total_machines_limit = 50
 # Specify the machine's instance type(for example, Standard_F2s_v2, Standard_DS3_v2)
 instance_type = <instance_type>
 
+# This boolean flag is used to indicate if we want to programmatically determine how many cores are used
+# per VM of the instance_type mentioned above.
+# NOTE: If enabled, this is a long call that takes ~ 1 minute to complete. It takes place at
+# the initialization of the machinery. If disabled, you need to specify the instance_type_cores below.
+find_number_of_cores_for_sku = true
+
+# The number of cores (vCPUs) that a VM of the instance_type mentioned above uses.
+# If find_number_of_cores_for_sku is enabled, this value will be ignored.
+# Set to 0 if you want to programmatically determine this value.
+# See note above. Otherwise, set to an integer.
+# For example for the instance_type Standard_F2s_v2, there are 2 cores per VM so the value for
+# instance_type_cores should be 2.
+instance_type_cores = 0
+
 # Specify the IP of the Result Server, as your virtual machine sees it.
 # It should be the nest ip address.
 resultserver_ip = <resultserver_ip>
@@ -53,9 +71,13 @@ storage_account_type = <storage_account_type>
 # Initial virtual machine pool size for each scale set
 initial_pool_size = 1
 
+# Reset pool size to initial_pool_size on CAPE restart
+reset_pool_size = true
+
 # Specify a comma-separated list of scale sets to be used, either available or to be created.
 # For each specified ID you have to define a dedicated section containing the details
 # about the respective scale set. (E.g. cuckoo1,cuckoo2,cuckoo3)
+# NOTE: NO SPACES
 scale_sets = cuckoo1
 
 # A percentage to be used for overprovisioning a scale set. To disable overprovisiong, set to 0
@@ -69,6 +91,10 @@ wait_time_to_reimage = 15
 # normal instances
 spot_instances = false
 
+# This boolean value is used to indicate if we want to wait for each VM to have its agent running before we
+# start pulling tasks off of the stack
+wait_for_agent_before_starting = true
+
 [cuckoo1]
 # The gallery image name to use when creating the virtual machine scale set.
 gallery_image_name = <gallery_image_name>
@@ -84,4 +110,4 @@ arch = x64
 
 # A tag used to specify on which guest scale set a sample should be run. All 
 # virtual machines in this scale set will have this tag
-tag = <tag>
+pool_tag = <tag>

conf/web.conf

@@ -38,6 +38,10 @@ hostname = https://127.0.0.1/
 ;hostname = https://www.capesandbox.com/
 # Check if config exists or try to extract before accept task as static
 check_config_exists = no
+# Assign architecture to task to fetch correct VM type
+dynamic_arch_determination = yes
+# Assign platform to task to fetch correct VM type
+dynamic_platform_determination = yes
 
 # ratelimit for anon users
 [ratelimit]

lib/cuckoo/common/abstracts.py

@@ -265,19 +265,20 @@ def availables(self):
         """
         return self.db.count_machines_available()
 
-    def acquire(self, machine_id=None, platform=None, tags=None):
+    def acquire(self, machine_id=None, platform=None, tags=None, arch=None):
         """Acquire a machine to start analysis.
         @param machine_id: machine ID.
         @param platform: machine platform.
         @param tags: machine tags
+        @param arch: machine arch
         @return: machine or None.
         """
         if machine_id:
             return self.db.lock_machine(label=machine_id)
         elif platform:
-            return self.db.lock_machine(platform=platform, tags=tags)
+            return self.db.lock_machine(platform=platform, tags=tags, arch=arch)
         else:
-            return self.db.lock_machine(tags=tags)
+            return self.db.lock_machine(tags=tags, arch=arch)
 
     def release(self, label=None):
         """Release a machine.

lib/cuckoo/common/web_utils.py

@@ -59,6 +59,8 @@
 
 db = Database()
 
+DYNAMIC_PLATFORM_DETERMINATION = web_cfg.general.dynamic_platform_determination
+
 HAVE_DIST = False
 # Distributed CAPE
 if repconf.distributed.enabled:
@@ -655,15 +657,16 @@ def download_file(**kwargs):
     if not kwargs.get("task_machines", []):
         kwargs["task_machines"] = [None]
 
-    platform = get_platform(magic_type)
+    if DYNAMIC_PLATFORM_DETERMINATION:
+        platform = get_platform(magic_type)
     if platform == "linux" and not linux_enabled and "Python" not in magic_type:
         return "error", {"error": "Linux binaries analysis isn't enabled"}
 
     if machine.lower() == "all":
         kwargs["task_machines"] = [vm.name for vm in db.list_machines(platform=platform)]
     elif machine:
         machine_details = db.view_machine(machine)
-        if hasattr(machine_details, "platform") and not machine_details.platform == platform:
+        if platform and hasattr(machine_details, "platform") and not machine_details.platform == platform:
             return "error", {"error": f"Wrong platform, {machine_details.platform} VM selected for {platform} sample"}
         else:
             kwargs["task_machines"] = [machine]
@@ -1084,9 +1087,9 @@ def force_bool(value):
     if not value:
         return False
 
-    if value in ("False", "false", "FALSE"):
+    if value.lower() in ("false", "no", "off", "0"):
         return False
-    elif value in ("True", "true", "TRUE"):
+    elif value.lower() in ("true", "yes", "on", "1"):
         return True
     else:
         log.warning("Value of %s cannot be converted from string to bool", value)