diff --git a/src/Config.php b/src/Config.php
index 3b3a7233..7cd9bd61 100644
--- a/src/Config.php
+++ b/src/Config.php
@@ -36,7 +36,14 @@ private function loadConfig($directory, $isUserDirectory = false)
     public function getCustomBinary($tool)
     {
         $binary = $this->path("{$tool}.binary");
-        return $binary ? escapePath($binary) : $binary;
+        if ($binary) {
+            $filename = basename($binary);
+            if (is_bool(strpos($filename, "{$tool}"))) {
+                throw new \RuntimeException("Invalid '{$tool}' binary ('{$tool}' not found in '{$binary}')");
+            }
+            return escapePath($binary);
+        }
+        return null;
     }
 
     public function value($path)
diff --git a/tests/Config/.phpqa.yml b/tests/Config/.phpqa.yml
index 3afcc0e4..d20db4e3 100644
--- a/tests/Config/.phpqa.yml
+++ b/tests/Config/.phpqa.yml
@@ -11,3 +11,11 @@ phpcpd:
 
 # Php file extensions to parse.
 extensions: php,inc,module
+
+# invalid binary
+phpunit:
+    binary: non-existent-binary
+
+# binary for other tool
+phpmetrics:
+    binary: ../../phpqa
diff --git a/tests/Config/ConfigTest.php b/tests/Config/ConfigTest.php
index 3160c830..c6c3163f 100644
--- a/tests/Config/ConfigTest.php
+++ b/tests/Config/ConfigTest.php
@@ -74,6 +74,22 @@ public function testUseCwdIfNoDirectoryIsSpecified()
         $config->loadUserConfig('');
     }
 
+    public function testThrowExceptionWhenBinaryDoesNotExist()
+    {
+        $config = new Config();
+        $config->loadUserConfig(__DIR__);
+        $this->shouldStopPhpqa();
+        $config->getCustomBinary('phpunit');
+    }
+
+    public function testThrowExceptionWhenWrongBinaryIsUsed()
+    {
+        $config = new Config();
+        $config->loadUserConfig(__DIR__);
+        $this->shouldStopPhpqa();
+        $config->getCustomBinary('phpmetrics');
+    }
+
     private function shouldStopPhpqa()
     {
         $this->setExpectedException('Exception');