From c1bafb534b83cddda432921a46782eb67edc2ce1 Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Mon, 30 Oct 2023 19:45:02 +0300 Subject: [PATCH 1/3] update selfupdate (#272) --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a7753a1..cea41d1 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/logrusorgru/aurora v2.0.3+incompatible github.com/microcosm-cc/bluemonday v1.0.25 github.com/miekg/dns v1.1.55 - github.com/minio/selfupdate v0.6.0 + github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/blackrock v0.0.1 github.com/projectdiscovery/fdmax v0.0.4 diff --git a/go.sum b/go.sum index 1d67e30..ccbd2ac 100644 --- a/go.sum +++ b/go.sum @@ -110,6 +110,8 @@ github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU= github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= +github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= +github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= From e65a43fc1ad2cea578a33e2c395e618b6f403fe9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 22:12:57 +0000 Subject: [PATCH 2/3] chore(deps): bump github.com/projectdiscovery/hmap from 0.0.20 to 0.0.23 Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.20 to 0.0.23. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.20...v0.0.23) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index cea41d1..745f4f9 100644 --- a/go.mod +++ b/go.mod @@ -90,7 +90,7 @@ require ( github.com/gorilla/css v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/gologger v1.1.11 - github.com/projectdiscovery/hmap v0.0.20 + github.com/projectdiscovery/hmap v0.0.23 github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 // indirect github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect golang.org/x/crypto v0.14.0 // indirect diff --git a/go.sum b/go.sum index ccbd2ac..ec8de8d 100644 --- a/go.sum +++ b/go.sum @@ -108,8 +108,6 @@ github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= -github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU= -github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -149,8 +147,8 @@ github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUd github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8= github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY= -github.com/projectdiscovery/hmap v0.0.20 h1:2W0TLRWNx3ACZo9Q60JsuAntQ8OprGAoe/4Fi5QdUHI= -github.com/projectdiscovery/hmap v0.0.20/go.mod h1:XI17aljoGOQhzcLq5iw8GKtSi5SmDTh0r5vRzq6dsJ0= +github.com/projectdiscovery/hmap v0.0.23 h1:tV/5gQuabE2nqDMS55vrd3HQYdwTuRJAm49nGu3DVl4= +github.com/projectdiscovery/hmap v0.0.23/go.mod h1:DYt1/UjEPA4vw6sk3PY8UB34ZnvXrDC3PQ+LBpkNlOA= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= From e2a42983f4c8015a675441faf685f6affaee4ae9 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Tue, 31 Oct 2023 20:43:56 +0530 Subject: [PATCH 3/3] revert #273 with explaination --- url/README.md | 7 +++++++ url/url.go | 6 +----- url/url_test.go | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/url/README.md b/url/README.md index b20b2da..ed71ed0 100644 --- a/url/README.md +++ b/url/README.md @@ -41,6 +41,13 @@ scanme.sh/%invalid/path - `.UpdateRelPath(newrelpath string, unsafe bool)` - `.Clone()` and more +- Dealing with Double URL Encoding of chars like `%0A` when `.Path` is directly updated + + when `url.Parse` is used to parse url like `https://127.0.0.1/%0A` it internally calls `u.setPath` which decodes `%0A` to `\n` and saves it in `u.Path` and when final url is created at time of writing to connection in http.Request Path is then escaped again thus `\n` becomes `%0A` and final url becomes `https://127.0.0.1/%0A` which is expected/required behavior. + + If `u.Path` is changed/updated directly after `url.Parse` ex: `u.Path = "%0A"` then at time of writing to connection in http.Request, Path is escaped again thus `%0A` becomes `%250A` and final url becomes `https://127.0.0.1/%250A` which is not expected/required behavior to avoid this we manually unescape/decode `u.Path` and we set `u.Path = unescape(u.Path)` which takes care of this edgecase. + + This is how `utils/url/URL` handles this edgecase when `u.Path` is directly updated. ### Note diff --git a/url/url.go b/url/url.go index b03867d..2c8ee66 100644 --- a/url/url.go +++ b/url/url.go @@ -100,7 +100,7 @@ func (u *URL) Clone() *URL { // String func (u *URL) String() string { var buff bytes.Buffer - if u.Scheme != "" { + if u.Scheme != "" && u.Host != "" { buff.WriteString(u.Scheme + "://") } if u.User != nil { @@ -308,10 +308,6 @@ func ParseURL(inputURL string, unsafe bool) (*URL, error) { } if u.IsRelative { return ParseRelativePath(inputURL, unsafe) - } else if unsafe { - // we are not relative, but we still need to call this in order to call - // the internal parser for paths url.Parse will not handle. - u.parseUnsafeRelativePath() } return u, nil } diff --git a/url/url_test.go b/url/url_test.go index 8f992fe..e3234a9 100644 --- a/url/url_test.go +++ b/url/url_test.go @@ -146,7 +146,7 @@ func TestParseInvalidUnsafe(t *testing.T) { for _, input := range testcases { u, err := ParseURL(input, true) require.Nilf(t, err, "got error for url %v", input) - require.Equal(t, input, u.String()) + require.Equal(t, input, u.URL.String()) } }