diff --git a/includes/class-simple-local-avatars.php b/includes/class-simple-local-avatars.php
index df0146e..8bd49a2 100644
--- a/includes/class-simple-local-avatars.php
+++ b/includes/class-simple-local-avatars.php
@@ -275,7 +275,7 @@ public function get_avatar_data( $args, $id_or_email ) {
 		}
 
 		// Local only mode
-		if ( ! $simple_local_avatar_url ) {
+		if ( ! $simple_local_avatar_url && ! empty( $this->options['only'] ) ) {
 			$args['url'] = $this->get_default_avatar_url( $args['size'] );
 		}