Fix HEAD method for robots.txt (#30603) (#30605)

Backport #30603 by @wxiaoguang

Fix #30601

```
~$ curl --head localhost:3000/robots.txt
HTTP/1.1 200 OK
Accept-Ranges: bytes
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 5
Content-Type: text/plain; charset=utf-8
Last-Modified: Wed, 19 Jul 2023 04:56:12 GMT
X-Gitea-Debug: RUN_MODE=dev
Date: Fri, 19 Apr 2024 12:59:44 GMT
```

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
(cherry picked from commit 7eaf7907d7f71e103baced018e6eeb271085789d)

routers/web/web.go

@@ -258,7 +258,7 @@ func Routes() *web.Route {
 		routes.Get("/metrics", append(mid, Metrics)...)
 	}
 
-	routes.Get("/robots.txt", append(mid, misc.RobotsTxt)...)
+	routes.Methods("GET,HEAD", "/robots.txt", append(mid, misc.RobotsTxt)...)
 	routes.Get("/ssh_info", misc.SSHInfo)
 	routes.Get("/api/healthz", healthcheck.Check)