diff --git a/CHANGES/623.bugfix.rst b/CHANGES/623.bugfix.rst
new file mode 100644
index 00000000..fb2d39d6
--- /dev/null
+++ b/CHANGES/623.bugfix.rst
@@ -0,0 +1 @@
+Changed call in ``with_port()`` to stop reencoding parts of the URL that were already encoded.
diff --git a/tests/test_url_update_netloc.py b/tests/test_url_update_netloc.py
index 1c9ba5c0..0dfc4303 100644
--- a/tests/test_url_update_netloc.py
+++ b/tests/test_url_update_netloc.py
@@ -202,6 +202,11 @@ def test_with_port_keeps_query_and_fragment():
     assert str(url.with_port(8888)) == "http://example.com:8888/?a=1#frag"
 
 
+def test_with_port_percent_encoded():
+    url = URL("http://user%name:pass%word@example.com/")
+    assert str(url.with_port(808)) == "http://user%25name:pass%25word@example.com:808/"
+
+
 def test_with_port_for_relative_url():
     with pytest.raises(ValueError):
         URL("path/to").with_port(1234)
diff --git a/yarl/_url.py b/yarl/_url.py
index e5c1e2ce..bb33fce3 100644
--- a/yarl/_url.py
+++ b/yarl/_url.py
@@ -883,9 +883,7 @@ def with_port(self, port):
         val = self._val
         return URL(
             self._val._replace(
-                netloc=self._make_netloc(
-                    val.username, val.password, val.hostname, port, encode=True
-                )
+                netloc=self._make_netloc(val.username, val.password, val.hostname, port)
             ),
             encoded=True,
         )