diff --git a/files/image_config/ntp/ntp.conf.j2 b/files/image_config/ntp/ntp.conf.j2
index 9b8b9264a396..a3b0f0909728 100644
--- a/files/image_config/ntp/ntp.conf.j2
+++ b/files/image_config/ntp/ntp.conf.j2
@@ -18,6 +18,7 @@ leapfile /usr/share/zoneinfo/leap-seconds.list
 {# Adding NTP servers. We need to know if we have some pools, to set proper
 config -#}
 {% set ns = namespace(is_pools=false) %}
+{% set ip_ver_ns = namespace(ipv4_server=false, ipv6_server=false) %}
 {% for server in NTP_SERVER if NTP_SERVER[server].admin_state != 'disabled' -%}
     {% set config = NTP_SERVER[server] -%}
     {# Server options -#}
@@ -56,6 +57,13 @@ config -#}
 restrict {{ resolve_as }} kod limited nomodify noquery
 {% endif %}
 
+{% if resolve_as | ipv4 -%}
+    {% set ip_ver_ns.ipv4_server = true %}
+{% elif resolve_as | ipv6 %}
+    {% set ip_ver_ns.ipv6_server = true %}
+{% endif -%}
+
+
 {% endfor -%}
 
 {% set trusted_keys_arr = [] -%}
@@ -125,9 +133,11 @@ interface listen {{ mgmt_prefix | ip }}
 {% endfor %}
 {% elif LOOPBACK_INTERFACE %}
 {% for (name, prefix) in LOOPBACK_INTERFACE|pfx_filter %}
-{% if prefix | ipv4 and name == 'Loopback0' %}
+{% if name == 'Loopback0' %}
+{% if prefix | ipv4 and ip_ver_ns.ipv4_server or prefix | ipv6 and ip_ver_ns.ipv6_server %}
 interface listen {{ prefix | ip }}
 {% endif %}
+{% endif %}
 {% endfor %}
 {% else %}
 interface listen eth0