Merge branch 'develop' into env_refactor/non_hypervised_env

.gitignore

@@ -66,7 +66,7 @@ yacctab.py
 data/
 
 # Install and build
-apps/rendering/resources/taskcollector/Release/*
+apps/rendering/resources/taskcollector/x64/Release/*
 apps/rendering/resources/taskcollector/taskcollector.vcxproj
 Installer/Installer_Win/deps/*
 Installer/Installer_Win/golem-*

Installer/Installer_Win/Golem.aip

@@ -407,14 +407,15 @@
     <ROW Name="aicustact.dll" SourcePath="&lt;AI_CUSTACTS&gt;aicustact.dll"/>
     <ROW Name="bannerfull01.jpg" SourcePath="Installer\Installer_Win\banner-full-01.jpg"/>
     <ROW Name="bannertop01.jpg" SourcePath="Installer\Installer_Win\banner-top-01.jpg"/>
+    <ROW Name="checkhypervinstallation.ps1" SourcePath="scripts\virtualization\check-hyperv-installation.ps1"/>
     <ROW Name="cleandockervm.ps1" SourcePath="scripts\docker\clean-docker-vm.ps1"/>
     <ROW Name="lzmaextractor.dll" SourcePath="&lt;AI_CUSTACTS&gt;lzmaextractor.dll"/>
     <ROW Name="preparedockerasadmin.ps1" SourcePath="scripts\docker\prepare-docker-as-admin.ps1"/>
     <ROW Name="tempFiles.dll" SourcePath="&lt;AI_CUSTACTS&gt;tempFiles.dll"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.MsiConditionComponent">
     <ROW Feature_="TileAssets_DefaultBuild" Level="4" Condition="VersionNT &lt; 603"/>
-    <ROW Feature_="DockerForWin" Level="1" Condition="(HYPER_V_AVAILABLE &lt;&gt; &quot;&quot;) AND (INSTALL_HYPERV = &quot;IDYES&quot;)"/>
+    <ROW Feature_="DockerForWin" Level="1" Condition="(HYPER_V_AVAILABLE = &quot;True&quot;) AND ((INSTALL_HYPERV = &quot;IDYES&quot;) OR (HYPER_V_INSTALLED = &quot;True&quot;))"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.MsiControlComponent">
     <ROW Dialog_="LicenseAgreementDlg" Control="AgreementText" Type="ScrollableText" X="20" Y="60" Width="330" Height="120" Attributes="7" Text="Installer\Installer_Win\LICENCE.rtf" Order="400" TextLocId="Control.Text.LicenseAgreementDlg#AgreementText_2" MsiKey="LicenseAgreementDlg#AgreementText"/>
@@ -469,10 +470,10 @@
     <ROW Action="AI_DATA_SETTER" Type="51" Source="AI_ExtractFiles" Target="[AI_SETUPEXEPATH]"/>
     <ROW Action="AI_DATA_SETTER_1" Type="51" Source="StopProcess_Golem" Target="Golem.exe"/>
     <ROW Action="AI_DATA_SETTER_10" Type="51" Source="CustomActionData" Target="AEYAbABhAGcAcwACADMAAQBTAGMAcgBpAHAAdABTAHIAYwBCAGkAbgBhAHIAeQBSAG8AdwBJAGQAAgBjAGwAZQBhAG4AZABvAGMAawBlAHIAdgBtAC4AcABzADE="/>
-    <ROW Action="AI_DATA_SETTER_11" Type="51" Source="CustomActionData" Target="AEkAcwA2ADQAQgBpAHQAAgAxAAEAUABhAHIAYQBtAHMAAgABAFMAYwByAGkAcAB0AAIAQQBJAF8AUwBlAHQATQBzAGkAUAByAG8AcABlAHIAdAB5ACAASABZAFAARQBSAF8AVgBfAEEAVgBBAEkATABBAEIATABFACAAKABHAGUAdAAtAFcAbQBpAE8AYgBqAGUAYwB0ACAALQBxAHUAZQByAHkAIAAiAHMAZQBsAGUAYwB0ACAAKgAgAGYAcgBvAG0AIABXAGkAbgAzADIAXwBPAHAAdABpAG8AbgBhAGwARgBlAGEAdAB1AHIAZQAgAHcAaABlAHIAZQAgAG4AYQBtAGUAIAA9ACAAJwBNAGkAYwByAG8AcwBvAGYAdAAtAEgAeQBwAGUAcgAtAFYAJwAiACk="/>
     <ROW Action="AI_DATA_SETTER_12" Type="51" Source="AI_RemoveAllTempFiles" Target="[AI_TEMP_FILE_ROLLBACK_INFO]"/>
     <ROW Action="AI_DATA_SETTER_13" Type="51" Source="CustomActionData" Target="AEYAbABhAGcAcwACADMAAQBTAGMAcgBpAHAAdABTAHIAYwBCAGkAbgBhAHIAeQBSAG8AdwBJAGQAAgBwAHIAZQBwAGEAcgBlAGQAbwBjAGsAZQByAGEAcwBhAGQAbQBpAG4ALgBwAHMAMQ=="/>
     <ROW Action="AI_DATA_SETTER_14" Type="51" Source="CustomActionData" Target="We have detected that you are trying to install Golem on Windows 10 Home. Unfortunately, your operating system does not support Hyper-V, but we have kept Docker Toolbox support.&#13;\n&#13;\nPlease download it manually from the official website https://docs.docker.com/toolbox/toolbox_install_windows/ in case you don&apos;t have it already installed on your computer. |Virtualization for Windows Home |MB_OK,MB_ICONINFORMATION,MB_DEFBUTTON1|BTN_PRESSED|[CLIENTPROCESSID]"/>
+    <ROW Action="AI_DATA_SETTER_15" Type="51" Source="CustomActionData" Target="AEYAbABhAGcAcwACADEAAQBTAGMAcgBpAHAAdABTAHIAYwBCAGkAbgBhAHIAeQBSAG8AdwBJAGQAAgBjAGgAZQBjAGsAaAB5AHAAZQByAHYAaQBuAHMAdABhAGwAbABhAHQAaQBvAG4ALgBwAHMAMQ=="/>
     <ROW Action="AI_DATA_SETTER_16" Type="51" Source="CustomActionData" Target="Since 0.19.0 Golem supports Hyper-V.&#13;\n&#13;\nPlease be aware that if you are using any other virtualization software (e.g. Docker Toolbox, VMware) do not to install Hyper-V as this may cause issues with other applications on your system. &#13;\n&#13;\nBy clicking &quot;Yes&quot; you are agreeing for Hyper-V to be installed. Otherwise please make sure you have Docker Toolbox installed before running Golem. |Hyper-V in Windows Professional |MB_YESNO,MB_ICONINFORMATION,MB_DEFBUTTON1|INSTALL_HYPERV|[CLIENTPROCESSID]"/>
     <ROW Action="AI_DATA_SETTER_2" Type="51" Source="StopProcess_Golemapp" Target="Golemapp.exe"/>
     <ROW Action="AI_DATA_SETTER_3" Type="51" Source="StopProcess_hyperg" Target="hyperg.exe"/>
@@ -513,7 +514,7 @@
     <ROW Action="AI_WinOptFeaturesRemove" Type="11265" Source="WinOptionalFeatures.dll" Target="OnWinOptFeaturesRemove" WithoutSeq="true"/>
     <ROW Action="AI_WinOptFeaturesRollback" Type="11521" Source="WinOptionalFeatures.dll" Target="OnWinOptFeaturesRollback" WithoutSeq="true"/>
     <ROW Action="AI_WinOptFeaturesUninstall" Type="1" Source="WinOptionalFeatures.dll" Target="OnWinOptFeaturesUninstall"/>
-    <ROW Action="CheckHyperVAvailability" Type="1" Source="PowerShellScriptLauncher.dll" Target="RunPowerShellScript" Options="1" AdditionalSeq="AI_DATA_SETTER_11"/>
+    <ROW Action="CheckHyperVInstallation" Type="1" Source="PowerShellScriptLauncher.dll" Target="RunPowerShellFileScript" Options="134217729" AdditionalSeq="AI_DATA_SETTER_15"/>
     <ROW Action="ExecuteUninstallDockerTb" Type="3122" Source="DOCKER_UNINSTALL_CLEAN" Target="/SILENT"/>
     <ROW Action="ExecuteUninstallPre011" Type="3122" Source="GOLEM_UNINSTALL_CLEAN" Target="/SILENT"/>
     <ROW Action="Execute_CleanDockerVM" Type="65" Source="PowerShellScriptLauncher.dll" Target="RunPowerShellFileScript" Options="134217729" AdditionalSeq="AI_DATA_SETTER_10"/>
@@ -602,23 +603,23 @@
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.MsiInstallUISequenceComponent">
     <ROW Action="AI_RESTORE_LOCATION" Condition="APPDIR=&quot;&quot;" Sequence="749"/>
-    <ROW Action="AI_ResolveKnownFolders" Sequence="55"/>
-    <ROW Action="AI_DpiContentScale" Sequence="54"/>
-    <ROW Action="AI_EnableDebugLog" Sequence="53"/>
+    <ROW Action="AI_ResolveKnownFolders" Sequence="53"/>
+    <ROW Action="AI_DpiContentScale" Sequence="52"/>
+    <ROW Action="AI_EnableDebugLog" Sequence="51"/>
     <ROW Action="AI_BACKUP_AI_SETUPEXEPATH" Sequence="99"/>
     <ROW Action="AI_RESTORE_AI_SETUPEXEPATH" Condition="AI_SETUPEXEPATH_ORIGINAL" Sequence="102"/>
     <ROW Action="AI_AppSearchEx" Sequence="101"/>
     <ROW Action="MessageBox_OldVersionDetected" Condition="( NOT Installed ) AND ( GOLEM_UNINSTALL &lt;&gt; &quot;&quot; )" Sequence="206"/>
     <ROW Action="AI_DATA_SETTER_9" Condition="( NOT Installed ) AND ( GOLEM_UNINSTALL &lt;&gt; &quot;&quot; )" Sequence="205"/>
     <ROW Action="ForceReboot" Condition="( Installed AND ( REMOVE = &quot;ALL&quot; OR AI_INSTALL_MODE = &quot;Remove&quot; ) )" Sequence="1301"/>
     <ROW Action="AI_ExtractTempFiles" Sequence="1001"/>
-    <ROW Action="MessageBox_HyperVUnavailable" Condition="( NOT Installed ) AND ( HYPER_V_AVAILABLE = &quot;&quot; )" Sequence="202"/>
-    <ROW Action="AI_DATA_SETTER_14" Condition="( NOT Installed ) AND ( HYPER_V_AVAILABLE = &quot;&quot; )" Sequence="201"/>
-    <ROW Action="CheckHyperVAvailability" Condition="( NOT Installed )" Sequence="52"/>
-    <ROW Action="AI_DATA_SETTER_11" Condition="( NOT Installed )" Sequence="51"/>
-    <ROW Action="MessageBox_DoYouWantHyperV" Condition="( NOT Installed ) AND ( HYPER_V_AVAILABLE &lt;&gt; &quot;&quot; )" Sequence="204"/>
-    <ROW Action="AI_DATA_SETTER_16" Condition="( NOT Installed ) AND ( HYPER_V_AVAILABLE &lt;&gt; &quot;&quot; )" Sequence="203"/>
+    <ROW Action="MessageBox_HyperVUnavailable" Condition="( NOT Installed ) AND ( HYPER_V_AVAILABLE &lt;&gt; &quot;True&quot; )" Sequence="202"/>
+    <ROW Action="AI_DATA_SETTER_14" Condition="( NOT Installed ) AND ( HYPER_V_AVAILABLE &lt;&gt; &quot;True&quot; )" Sequence="201"/>
+    <ROW Action="MessageBox_DoYouWantHyperV" Condition="( NOT Installed ) AND ( HYPER_V_AVAILABLE = &quot;True&quot; AND HYPER_V_INSTALLED &lt;&gt; &quot;True&quot; )" Sequence="204"/>
+    <ROW Action="AI_DATA_SETTER_16" Condition="( NOT Installed ) AND ( HYPER_V_AVAILABLE = &quot;True&quot; AND HYPER_V_INSTALLED &lt;&gt; &quot;True&quot; )" Sequence="203"/>
     <ROW Action="SetRunningUser" Condition="( NOT Installed )" Sequence="1299"/>
+    <ROW Action="CheckHyperVInstallation" Condition="( NOT Installed )" Sequence="77"/>
+    <ROW Action="AI_DATA_SETTER_15" Condition="( NOT Installed )" Sequence="76"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.MsiLaunchConditionsComponent">
     <ROW Condition="( Version9X OR ( NOT VersionNT64 ) OR ( VersionNT64 AND ((VersionNT64 &lt;&gt; 502) OR (ServicePackLevel &lt;&gt; 2) OR (MsiNTProductType &lt;&gt; 1)) AND ((VersionNT64 &lt;&gt; 502) OR (ServicePackLevel &lt;&gt; 2) OR (MsiNTProductType = 1)) AND ((VersionNT64 &lt;&gt; 600) OR (MsiNTProductType &lt;&gt; 1)) AND ((VersionNT64 &lt;&gt; 600) OR (MsiNTProductType = 1)) AND ((VersionNT64 &lt;&gt; 601) OR (MsiNTProductType &lt;&gt; 1)) AND ((VersionNT64 &lt;&gt; 601) OR (MsiNTProductType = 1)) AND ((VersionNT64 &lt;&gt; 602) OR (MsiNTProductType &lt;&gt; 1)) AND ((VersionNT64 &lt;&gt; 602) OR (MsiNTProductType = 1)) AND ((VersionNT64 &lt;&gt; 603) OR (MsiNTProductType &lt;&gt; 1)) AND ((VersionNT64 &lt;&gt; 603) OR (MsiNTProductType = 1)) AND ((VersionNT64 &lt;&gt; 1000) OR (MsiNTProductType &lt;&gt; 1) OR (WindowsBuild &lt;&gt; 10240)) AND ((VersionNT64 &lt;&gt; 1000) OR (MsiNTProductType &lt;&gt; 1) OR (WindowsBuild &lt;&gt; 10586)) AND ((VersionNT64 &lt;&gt; 1000) OR (MsiNTProductType &lt;&gt; 1) OR (WindowsBuild &lt;&gt; 14393)) AND ((VersionNT64 &lt;&gt; 1000) OR (MsiNTProductType &lt;&gt; 1) OR (WindowsBuild &lt;&gt; 15063)) ) )" Description="[ProductName] can not be installed on the following Windows versions: [WindowsTypeNT64Display]." DescriptionLocId="AI.LaunchCondition.NoSpecificNT64" IsPredefined="true" Builds="DefaultBuild"/>

golem/core/keysauth.py

@@ -11,7 +11,7 @@
 from eth_utils import encode_hex, decode_hex
 from golem_messages.cryptography import ECCx, mk_privkey, ecdsa_verify, \
     privtopub
-
+from golem_messages.utils import pubkey_to_address
 
 logger = logging.getLogger(__name__)
 
@@ -85,6 +85,7 @@ def __init__(self, datadir: str, private_key_name: str, password: str,
         self.ecc = ECCx(prv)
         self.public_key = pub
         self.key_id = encode_hex(pub)[2:]
+        self.eth_addr = pubkey_to_address(pub)
         self.difficulty = KeysAuth.get_difficulty(self.key_id)
 
     @staticmethod

golem/database/database.py

@@ -55,7 +55,7 @@ def execute_sql(self, sql, params=None, require_commit=True):
 
 
 class Database:
-    SCHEMA_VERSION = 28
+    SCHEMA_VERSION = 29
 
     def __init__(self,  # noqa pylint: disable=too-many-arguments
                  db: peewee.Database,

golem/database/schemas/018_schema.py

@@ -1,7 +1,7 @@
 # pylint: disable=no-member
 # pylint: disable=unused-argument
 import peewee as pw
-from golem.utils import pubkeytoaddr
+from golem_messages.utils import pubkey_to_address
 
 SCHEMA_VERSION = 18
 
@@ -19,7 +19,7 @@ def _fill_payer_address(database):
             break
         for entry in entries:
             sender_node, subtask = entry
-            payer_address = pubkeytoaddr(sender_node)[2:]
+            payer_address = pubkey_to_address(sender_node)[2:]
             database.execute_sql(
                 "UPDATE income SET payer_address = ?"
                 " WHERE sender_node = ? AND subtask = ?",

golem/database/schemas/029_wallet_operation_type.py

@@ -0,0 +1,16 @@
+# pylint: disable=no-member
+# pylint: disable=unused-argument
+import peewee as pw
+
+SCHEMA_VERSION = 29
+
+
+def migrate(migrator, database, fake=False, **kwargs):
+    migrator.add_fields(
+        'walletoperation',
+        operation_type=pw.CharField(default='task_payment'),
+    )
+
+
+def rollback(migrator, database, fake=False, **kwargs):
+    migrator.remove_fields('walletoperation', 'task_payment')

golem/interface/client/account.py

@@ -1,7 +1,11 @@
 import datetime
 import getpass
 import sys
-from typing import Dict, Any
+from typing import (
+    Any,
+    Dict,
+    TYPE_CHECKING,
+)
 
 from decimal import Decimal
 from ethereum.utils import denoms
@@ -11,13 +15,17 @@
 from golem.core.deferred import sync_wait
 from golem.interface.command import Argument, command, group
 
+if TYPE_CHECKING:
+    # pylint: disable=unused-import
+    from golem.rpc.session import ClientProxy
+
 MIN_LENGTH = 5
 MIN_SCORE = 2
 
 
 @group(help="Manage account")
 class Account:
-    client = None
+    client: 'ClientProxy'
 
     amount_arg = Argument('amount', help='Amount to withdraw, eg 1.45')
     address_arg = Argument('destination', help='Address to send the funds to')
@@ -37,7 +45,8 @@ def info(self) -> Dict[str, Any]:  # pylint: disable=no-self-use
 
         computing_trust = sync_wait(client.get_computing_trust(node_key))
         requesting_trust = sync_wait(client.get_requesting_trust(node_key))
-        payment_address = sync_wait(client.get_payment_address())
+        # pylint: disable=protected-access
+        payment_address = sync_wait(client._call('pay.ident'))
 
         balance = sync_wait(client.get_balance())
 

golem/task/taskserver.py

@@ -25,8 +25,8 @@
 from apps.appsmanager import AppsManager
 from apps.core.task.coretask import CoreTask
 from golem.clientconfigdescriptor import ClientConfigDescriptor
-from golem.core.variables import MAX_CONNECT_SOCKET_ADDRESSES
 from golem.core.common import node_info_str, short_node_id
+from golem.core.variables import MAX_CONNECT_SOCKET_ADDRESSES
 from golem.environments.environment import SupportStatus, UnsupportReason
 from golem.envs.docker.cpu import DockerCPUConfig
 from golem.envs.docker.non_hypervised import NonHypervisedDockerCPUEnvironment
@@ -56,7 +56,6 @@
 from golem.task.taskconnectionshelper import TaskConnectionsHelper
 from golem.task.taskstate import TaskOp
 from golem.utils import decode_hex
-
 from .server import concent
 from .server import helpers
 from .server import queue_ as srv_queue
@@ -67,7 +66,6 @@
 from .taskmanager import TaskManager
 from .tasksession import TaskSession
 
-
 logger = logging.getLogger(__name__)
 
 tmp_cycler = itertools.cycle(list(range(550)))
@@ -331,7 +329,7 @@ def _request_task(self, theader: dt_tasks.TaskHeader) -> Optional[str]:
                 max_memory_size=self.config_desc.max_memory_size,
                 concent_enabled=self.client.concent_service.enabled,
                 provider_public_key=self.get_key_id(),
-                provider_ethereum_public_key=self.get_key_id(),
+                provider_ethereum_address=self.keys_auth.eth_addr,
                 task_header=theader,
             )
             msg_queue.put(

golem/utils.py

@@ -16,10 +16,6 @@ def find_free_net_port():
     return s.getsockname()[1]  # Return the port assigned.
 
 
-def pubkeytoaddr(pubkey: str) -> str:
-    return to_checksum_address(encode_hex(sha3(decode_hex(pubkey))[12:]))
-
-
 def privkeytoaddr(privkey: bytes) -> str:
     """
     Converts a private key bytes sequence to a string, representing the

requirements.txt

@@ -1,4 +1,4 @@
-#git+https://github.com/golemfactory/golem-messages@<branchname>
+#git+https://github.com/golemfactory/golem-messages@<branchname>#egg=golem-messages
 --extra-index-url https://builds.golem.network --trusted-host build.golem.network
 appdirs==1.4.3
 argh==0.26.2
@@ -36,7 +36,7 @@ eth-tester==0.1.0-beta.24
 eth-utils==1.0.3
 ethereum==1.6.1
 fs==2.4.4
-Golem-Messages==3.6.0
+Golem-Messages==3.7.0
 Golem-Smart-Contracts-Interface==1.10.0
 greenlet==0.4.15
 h2==3.0.1

requirements_to-freeze.txt

@@ -1,4 +1,4 @@
-#git+https://github.com/golemfactory/golem-messages@<branchname>
+#git+https://github.com/golemfactory/golem-messages@<branchname>#egg=golem-messages
 --extra-index-url https://builds.golem.network
 appdirs>=1.4
 asn1crypto==0.22.0
@@ -16,7 +16,7 @@ docker==3.5.0
 enforce==0.3.4
 eth-utils==1.0.3
 ethereum==1.6.1
-Golem-Messages==3.6.0
+Golem-Messages==3.7.0
 Golem-Smart-Contracts-Interface==1.10.0
 html2text==2018.1.9
 humanize==0.5.1

scripts/concent_acceptance_tests/additional_verification/base.py

@@ -6,6 +6,7 @@
 from golem_messages import factories as msg_factories
 from golem_messages.message import tasks as tasks_msg
 
+
 from apps.blender.blenderenvironment import BlenderEnvironment
 
 from golem.core.simplehash import SimpleHash
@@ -77,32 +78,43 @@ def get_ctd(self, **kwargs):
         )
         return ctd
 
-    def get_srv_file_kwargs(self, results_filename=None):
-        if not results_filename:
-            results_filename = self.results_filename
-
-        rct_path = 'subtask_results_rejected__report_computed_task__'
-        ttc_path = rct_path + 'task_to_compute__'
+    def ttc_file_kwargs(self, ttc_path = ''):
         return {
             ttc_path + 'compute_task_def': self.get_ctd(),
             ttc_path + 'size': self.size(self.resources_filename),
             ttc_path + 'package_hash': self.hash(self.resources_filename),
             ttc_path + 'concent_enabled': True,
+        }
+
+    def rct_file_kwargs(self, results_filename=None):
+        if not results_filename:
+            results_filename = self.results_filename
+
+        rct_path = 'subtask_results_rejected__report_computed_task__'
+        return {
             rct_path + 'size': self.size(results_filename),
             rct_path + 'package_hash': self.hash(results_filename),
         }
 
     def get_srv(self, results_filename=None, **kwargs):
         rct_path = 'subtask_results_rejected__report_computed_task__'
-        files_kwargs = self.get_srv_file_kwargs(
-            results_filename=results_filename)
-        files_kwargs.update(kwargs)
-        return msg_factories.concents.SubtaskResultsVerifyFactory(
+        files_kwargs = self.rct_file_kwargs(results_filename=results_filename)
+
+        ttc = self.gen_ttc(**self.ttc_file_kwargs())
+
+        srv = msg_factories.concents.SubtaskResultsVerifyFactory(
             **self.gen_rtc_kwargs(rct_path),
-            **self.gen_ttc_kwargs(rct_path + 'task_to_compute__'),
+            **{rct_path + 'task_to_compute': ttc},
             subtask_results_rejected__sign__privkey=self.requestor_priv_key,
             **files_kwargs,
+            **kwargs,
+        )
+        srv.sign_concent_promissory_note(
+            deposit_contract_address=
+            self.ethereum_config.deposit_contract_address,
+            private_key=self.provider_priv_key,
         )
+        return srv
 
     def get_correct_srv(self, results_filename=None, **kwargs):
         vn = tasks_msg.SubtaskResultsRejected.REASON.VerificationNegative