From da4da1b74d1962e029e586f222ec0ad5b2f93475 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:36:08 +0530 Subject: [PATCH 01/12] add process for conformance suite --- .../test/scala/zio/http/ConformanceSpec.scala | 1446 +++++++++++++++++ 1 file changed, 1446 insertions(+) create mode 100644 zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala new file mode 100644 index 0000000000..e12913b7cf --- /dev/null +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala @@ -0,0 +1,1446 @@ +package zio.http + +import java.time.format.DateTimeFormatter +import java.time.ZonedDateTime + +import zio._ +import zio.test.Assertion._ +import zio.test.TestAspect._ +import zio.test._ + +import zio.http._ + +object ConformanceSpec extends ZIOSpecDefault { + + /** + * This test suite is inspired by and built upon the findings from the + * research paper: "Who's Breaking the Rules? Studying Conformance to the HTTP + * Specifications and its Security Impact" by Jannis Rautenstrauch and Ben + * Stock, presented at the 19th ACM Asia Conference on Computer and + * Communications Security (ASIA CCS) 2024. + * + * Paper URL: https://doi.org/10.1145/3634737.3637678 + * GitHub Project: https://github.com/cispa/http-conformance + * + */ + + val validUrl = URL.decode("http://example.com").toOption.getOrElse(URL.root) + + override def spec = + suite("ConformanceSpec")( + suite("Statuscodes")( + test("should not send body for 204 No Content responses(code_204_no_additional_content)") { + val app = Routes( + Method.GET / "no-content" -> Handler.fromResponse( + Response.status(Status.NoContent), + ), + ) + + val request = Request.get("/no-content") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.NoContent, + response.body.isEmpty, + ) + }, + test("should not send body for 205 Reset Content responses(code_205_no_content_allowed)") { + val app = Routes( + Method.GET / "reset-content" -> Handler.fromResponse( + Response.status(Status.ResetContent), + ), + ) + + val request = Request.get("/reset-content") + + for { + response <- app.runZIO(request) + } yield assertTrue(response.status == Status.ResetContent, response.body.isEmpty) + }, + test("should include Content-Range for 206 Partial Content response(code_206_content_range)") { + val app = Routes( + Method.GET / "partial" -> Handler.fromResponse( + Response + .status(Status.PartialContent) + .addHeader(Header.ContentRange.StartEnd("bytes", 0, 14)), + ), + ) + + val request = Request.get("/partial") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.PartialContent, + response.headers.contains(Header.ContentRange.name), + ) + }, + test( + "should not include Content-Range in header for multipart/byteranges response(code_206_content_range_of_multiple_part_response)", + ) { + val boundary = zio.http.Boundary("A12345") + + val app = Routes( + Method.GET / "partial" -> Handler.fromResponse( + Response + .status(Status.PartialContent) + .addHeader(Header.ContentType(MediaType("multipart", "byteranges"), Some(boundary))), + ), + ) + + val request = Request.get("/partial") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.PartialContent, + !response.headers.contains(Header.ContentRange.name), + response.headers.contains(Header.ContentType.name), + ) + }, + test("should include necessary headers in 206 Partial Content response(code_206_headers)") { + val app = Routes( + Method.GET / "partial" -> Handler.fromResponse( + Response + .status(Status.PartialContent) + .addHeader(Header.ETag.Strong("abc")) + .addHeader(Header.CacheControl.MaxAge(3600)), + ), + Method.GET / "full" -> Handler.fromResponse( + Response + .status(Status.Ok) + .addHeader(Header.ETag.Strong("abc")) + .addHeader(Header.CacheControl.MaxAge(3600)), + ), + ) + + val requestWithRange = + Request.get("/partial").addHeader(Header.Range.Single("bytes", 0, Some(14))) + val requestWithoutRange = Request.get("/full") + + for { + responseWithRange <- app.runZIO(requestWithRange) + responseWithoutRange <- app.runZIO(requestWithoutRange) + } yield assertTrue( + responseWithRange.status == Status.PartialContent, + responseWithRange.headers.contains(Header.ETag.name), + responseWithRange.headers.contains(Header.CacheControl.name), + responseWithoutRange.status == Status.Ok, + ) + }, + test("should include WWW-Authenticate header for 401 Unauthorized response(code_401_www_authenticate)") { + val app = Routes( + Method.GET / "unauthorized" -> Handler.fromResponse( + Response + .status(Status.Unauthorized) + .addHeader(Header.WWWAuthenticate.Basic(Some("simple"))), + ), + ) + + val request = Request.get("/unauthorized") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.Unauthorized, + response.headers.contains(Header.WWWAuthenticate.name), + ) + }, + test("should include Allow header for 405 Method Not Allowed response(code_405_allow)") { + val app = Routes( + Method.POST / "not-allowed" -> Handler.fromResponse( + Response + .status(Status.Ok), + ), + ) + + val request = Request.get("/not-allowed") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.MethodNotAllowed, + response.headers.contains(Header.Allow.name), + ) + }, + test( + "should include Proxy-Authenticate header for 407 Proxy Authentication Required response(code_407_proxy_authenticate)", + ) { + val app = Routes( + Method.GET / "proxy-auth" -> Handler.fromResponse( + Response + .status(Status.ProxyAuthenticationRequired) + .addHeader( + Header.ProxyAuthenticate(Header.AuthenticationScheme.Basic, Some("proxy")), + ), + ), + ) + + val request = Request.get("/proxy-auth") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.ProxyAuthenticationRequired, + response.headers.contains(Header.ProxyAuthenticate.name), + ) + }, + test("should return 304 without content(code_304_no_content)") { + val app = Routes( + Method.GET / "no-content" -> Handler.fromResponse( + Response + .status(Status.NotModified) + .copy(body = Body.empty), + ), + ) + + val request = Request.get("/no-content") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.NotModified, + response.body.isEmpty, + ) + }, + test("should return 304 with correct headers(code_304_headers)") { + val headers = Headers( + Header.ETag.Strong("abc"), + Header.CacheControl.MaxAge(3600), + Header.Vary("Accept-Encoding"), + ) + + val app = Routes( + Method.GET / "with-headers" -> Handler.fromResponse( + Response + .status(Status.NotModified) + .addHeaders(headers), + ), + ) + + val request = Request.get("/with-headers") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.NotModified, + response.headers.contains(Header.ETag.name), + response.headers.contains(Header.CacheControl.name), + response.headers.contains(Header.Vary.name), + ) + }, + test("should include Location header in 300 MULTIPLE CHOICES response(code_300_location)") { + val testUrl = URL.decode("/People.html#tim").toOption.getOrElse(URL.root) + + val validResponse = Response + .status(Status.MultipleChoices) + .addHeader(Header.Location(testUrl)) + + val invalidResponse = Response + .status(Status.MultipleChoices) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.MultipleChoices, + responseValid.headers.contains(Header.Location.name), + responseInvalid.status == Status.MultipleChoices, + !responseInvalid.headers.contains(Header.Location.name), + ) + }, + test("300 MULTIPLE CHOICES response should have body content(code_300_metadata)") { + val validResponse = Response + .status(Status.MultipleChoices) + .copy(body = Body.fromString("
ABC
")) + + val invalidResponse = Response + .status(Status.MultipleChoices) + .copy(body = Body.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + validBody <- responseValid.body.asString + responseInvalid <- app.runZIO(Request.get("/invalid")) + invalidBody <- responseInvalid.body.asString + + } yield assertTrue( + responseValid.status == Status.MultipleChoices, + validBody.contains("ABC"), + responseInvalid.status == Status.MultipleChoices, + invalidBody.isEmpty, + ) + }, + test("should not require body content for HEAD requests(code_300_metadata)") { + val response = Response + .status(Status.MultipleChoices) + .copy(body = Body.empty) + val app = Routes( + Method.HEAD / "head" -> Handler.fromResponse(response), + ) + + for { + headResponse <- app.runZIO(Request.head("/head")) + } yield assertTrue( + headResponse.status == Status.MultipleChoices, + headResponse.body.isEmpty, + ) + }, + test("should include Location header in 301 MOVED PERMANENTLY response(code_301_location)") { + + val validResponse = Response + .status(Status.MovedPermanently) + .addHeader(Header.Location(validUrl)) + + val invalidResponse = Response + .status(Status.MovedPermanently) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.MovedPermanently, + responseValid.headers.contains(Header.Location.name), + responseInvalid.status == Status.MovedPermanently, + !responseInvalid.headers.contains(Header.Location.name), + ) + }, + test("should include Location header in 302 FOUND response(code_302_location)") { + + val validResponse = Response + .status(Status.Found) + .addHeader(Header.Location(validUrl)) + + val invalidResponse = Response + .status(Status.Found) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.Found, + responseValid.headers.contains(Header.Location.name), + responseInvalid.status == Status.Found, + !responseInvalid.headers.contains(Header.Location.name), + ) + }, + test("should include Location header in 303 SEE OTHER response(code_303_location)") { + + val validResponse = Response + .status(Status.SeeOther) + .addHeader(Header.Location(validUrl)) + + val invalidResponse = Response + .status(Status.SeeOther) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.SeeOther, + responseValid.headers.contains(Header.Location.name), + responseInvalid.status == Status.SeeOther, + !responseInvalid.headers.contains(Header.Location.name), + ) + }, + test("should include Location header in 307 TEMPORARY REDIRECT response(code_307_location)") { + + val validResponse = Response + .status(Status.TemporaryRedirect) + .addHeader(Header.Location(validUrl)) + + val invalidResponse = Response + .status(Status.TemporaryRedirect) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.TemporaryRedirect, + responseValid.headers.contains(Header.Location.name), + responseInvalid.status == Status.TemporaryRedirect, + !responseInvalid.headers.contains(Header.Location.name), + ) + }, + test("should include Location header in 308 PERMANENT REDIRECT response(code_308_location)") { + + val validResponse = Response + .status(Status.PermanentRedirect) + .addHeader(Header.Location(validUrl)) + + val invalidResponse = Response + .status(Status.PermanentRedirect) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.PermanentRedirect, + responseValid.headers.contains(Header.Location.name), + responseInvalid.status == Status.PermanentRedirect, + !responseInvalid.headers.contains(Header.Location.name), + ) + }, + test( + "should include Retry-After header in 413 Content Too Large response if condition is temporary (code_413_retry_after)", + ) { + val validResponse = Response + .status(Status.RequestEntityTooLarge) + .addHeader(Header.RetryAfter.ByDuration(10.seconds)) + + val invalidResponse = Response + .status(Status.RequestEntityTooLarge) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.RequestEntityTooLarge, + responseValid.headers.contains(Header.RetryAfter.name), + responseInvalid.status == Status.RequestEntityTooLarge, + !responseInvalid.headers.contains(Header.RetryAfter.name), + ) + }, + test( + "should include Accept or Accept-Encoding header in 415 Unsupported Media Type response (code_415_unsupported_media_type)", + ) { + val validResponse = Response + .status(Status.UnsupportedMediaType) + .addHeader(Header.Accept(MediaType.application.json)) + + val invalidResponse = Response + .status(Status.UnsupportedMediaType) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.UnsupportedMediaType, + responseValid.headers.contains(Header.Accept.name) || + responseValid.headers.contains(Header.AcceptEncoding.name), + responseInvalid.status == Status.UnsupportedMediaType, + !responseInvalid.headers.contains(Header.Accept.name) && + !responseInvalid.headers.contains(Header.AcceptEncoding.name), + ) + }, + test("should include Content-Range header in 416 Range Not Satisfiable response (code_416_content_range)") { + val validResponse = Response + .status(Status.RequestedRangeNotSatisfiable) + .addHeader(Header.ContentRange.RangeTotal("bytes", 47022)) + + val invalidResponse = Response + .status(Status.RequestedRangeNotSatisfiable) + .addHeader(Header.Custom("Content-Range", ",;")) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.RequestedRangeNotSatisfiable, + responseValid.headers.contains(Header.ContentRange.name), + responseInvalid.status == Status.RequestedRangeNotSatisfiable, + responseInvalid.headers.contains(Header.ContentRange.name), + responseInvalid.headers.get(Header.ContentRange.name).contains(",;"), + ) + }, + ), + suite("HTTP Headers")( + suite("code_400_after_bad_host_request")( + test("should return 200 OK if Host header is present") { + val route = Method.GET / "test" -> Handler.ok + val app = Routes(route) + val requestWithHost = Request.get("/test").addHeader(Header.Host("localhost")) + for { + response <- app.runZIO(requestWithHost) + } yield assertTrue(response.status == Status.Ok) + }, + test("should return 400 Bad Request if Host header is missing") { + val route = Method.GET / "test" -> Handler.ok + val app = Routes(route) + val requestWithoutHost = Request.get("/test") + + for { + response <- app.runZIO(requestWithoutHost) + } yield assertTrue(response.status == Status.BadRequest) + }, + test("should return 400 Bad Request if there are multiple Host headers") { + val route = Method.GET / "test" -> Handler.ok + val app = Routes(route) + val requestWithTwoHosts = Request + .get("/test") + .addHeader(Header.Host("example.com")) + .addHeader(Header.Host("another.com")) + + for { + response <- app.runZIO(requestWithTwoHosts) + } yield assertTrue(response.status == Status.BadRequest) + }, + test("should return 400 Bad Request if Host header is invalid") { + val route = Method.GET / "test" -> Handler.ok + val app = Routes(route) + val requestWithInvalidHost = Request + .get("/test") + .addHeader(Header.Host("invalid_host")) + + for { + response <- app.runZIO(requestWithInvalidHost) + } yield assertTrue(response.status == Status.BadRequest) + }, + ), + test("should not include Content-Length header for 2XX CONNECT responses(content_length_2XX_connect)") { + val app = Routes( + Method.CONNECT / "" -> Handler.fromResponse( + Response.status(Status.Ok), + ), + ) + + val decodedUrl = URL.decode("https://example.com:443") + + val request = decodedUrl match { + case Right(url) => Request(method = Method.CONNECT, url = url) + case Left(_) => throw new RuntimeException("Failed to decode the URL") + } + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.Ok, + !response.headers.contains(Header.ContentLength.name), + ) + }, + test("should not include Transfer-Encoding header for 2XX CONNECT responses(transfer_encoding_2XX_connect)") { + val app = Routes( + Method.CONNECT / "" -> Handler.fromResponse( + Response.status(Status.Ok), + ), + ) + + val decodedUrl = URL.decode("https://example.com:443") + + val request = decodedUrl match { + case Right(url) => Request(method = Method.CONNECT, url = url) + case Left(_) => throw new RuntimeException("Failed to decode the URL") + } + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.Ok, + !response.headers.contains(Header.TransferEncoding.name), + ) + }, + test("should not return overly detailed Server header(server_header_long)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Server", "SimpleServer")) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Server", "a" * 101)) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield { + assertTrue( + responseValid.headers.get(Header.Server.name).exists(_.length <= 100), + responseInvalid.headers.get(Header.Server.name).exists(_.length > 100), + ) + } + }, + test("should include Content-Type header for responses with content(content_type_header_required)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.ContentType(MediaType.text.html)) + .copy(body = Body.fromString("
ABC
")) + + val invalidResponse = Response + .status(Status.Ok) + .copy(body = Body.fromString("
ABC
")) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield { + assertTrue( + responseValid.headers.contains(Header.ContentType.name), + !responseInvalid.headers.contains(Header.ContentType.name), + ) + } + }, + test("should include Accept-Patch header when PATCH is supported(accept_patch_presence)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.AcceptPatch(NonEmptyChunk(MediaType.application.json))) + + val invalidResponse = Response + .status(Status.Ok) + .copy(headers = Headers.empty) + + val app = Routes( + Method.OPTIONS / "valid" -> Handler.fromResponse(validResponse), + Method.OPTIONS / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.options("/valid")) + responseInvalid <- app.runZIO(Request.options("/invalid")) + } yield { + assertTrue( + responseValid.headers.contains(Header.AcceptPatch.name), + !responseInvalid.headers.contains(Header.AcceptPatch.name), + ) + } + }, + test("should include Date header in responses (date_header_required)") { + val validDate = ZonedDateTime.parse("Thu, 20 Mar 2025 20:03:00 GMT", DateTimeFormatter.RFC_1123_DATE_TIME) + + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.Date(validDate)) + + val invalidResponse = Response + .status(Status.Ok) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.headers.contains(Header.Date.name), + !responseInvalid.headers.contains(Header.Date.name), + ) + }, + suite("CSP Header")( + test("should not send more than one CSP header (duplicate_csp)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.ContentSecurityPolicy.defaultSrc(Header.ContentSecurityPolicy.Source.Self)) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.ContentSecurityPolicy.defaultSrc(Header.ContentSecurityPolicy.Source.Self)) + .addHeader(Header.ContentSecurityPolicy.imgSrc(Header.ContentSecurityPolicy.Source.Self)) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield { + val cspHeadersValid = responseValid.headers.toList.collect { + case h if h.headerName == Header.ContentSecurityPolicy.name => h + } + val cspHeadersInvalid = responseInvalid.headers.toList.collect { + case h if h.headerName == Header.ContentSecurityPolicy.name => h + } + + assertTrue( + cspHeadersValid.length == 1, + cspHeadersInvalid.length > 1, + ) + } + }, + // Note: Content-Security-Policy-Report-Only Header to be Supported + ), + ), + suite("sts")( + // Note: Strict-Transport-Security Header to be Supported + + ), + suite("Transfer-Encoding")( + suite("no_transfer_encoding_1xx_204")( + test("should return valid when Transfer-Encoding is not present for 1xx or 204 status") { + val app = Routes( + Method.GET / "no-content" -> Handler.fromResponse( + Response.status(Status.NoContent), + ), + Method.GET / "continue" -> Handler.fromResponse( + Response.status(Status.Continue), + ), + ) + for { + responseNoContent <- app.runZIO(Request.get("/no-content")) + responseContinue <- app.runZIO(Request.get("/continue")) + } yield assertTrue(responseNoContent.status == Status.NoContent) && + assertTrue(!responseNoContent.headers.contains(Header.TransferEncoding.name)) && + assertTrue(responseContinue.status == Status.Continue) && + assertTrue(!responseContinue.headers.contains(Header.TransferEncoding.name)) + }, + test("should return invalid when Transfer-Encoding is present for 1xx or 204 status") { + val app = Routes( + Method.GET / "no-content" -> Handler.fromResponse( + Response.status(Status.NoContent).addHeader(Header.TransferEncoding.Chunked), + ), + Method.GET / "continue" -> Handler.fromResponse( + Response.status(Status.Continue).addHeader(Header.TransferEncoding.Chunked), + ), + ) + + for { + responseNoContent <- app.runZIO(Request.get("/no-content")) + responseContinue <- app.runZIO(Request.get("/continue")) + } yield assertTrue(responseNoContent.status == Status.NoContent) && + assertTrue(responseNoContent.headers.contains(Header.TransferEncoding.name)) && + assertTrue(responseContinue.status == Status.Continue) && + assertTrue(responseContinue.headers.contains(Header.TransferEncoding.name)) + }, + ), + suite("transfer_encoding_http11")( + test("should not send Transfer-Encoding in response if request HTTP version is below 1.1") { + val app = Routes( + Method.GET / "test" -> Handler.fromResponse( + Response.ok.addHeader(Header.TransferEncoding.Chunked), + ), + ) + + val request = Request.get("/test").copy(version = Version.`HTTP/1.0`) + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.Ok, + !response.headers.contains(Header.TransferEncoding.name), + ) + }, + test("should send Transfer-Encoding in response if request HTTP version is 1.1 or higher") { + val app = Routes( + Method.GET / "test" -> Handler.fromResponse( + Response.ok.addHeader(Header.TransferEncoding.Chunked), + ), + ) + + val request = Request.get("/test").copy(version = Version.`HTTP/1.1`) + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.Ok, + response.headers.contains(Header.TransferEncoding.name), + ) + }, + ), + ), + suite("HTTP-Methods")( + test("should not send body for HEAD requests(content_head_request)") { + val route = Routes( + Method.GET / "test" -> Handler.fromResponse(Response.text("This is the body")), + Method.HEAD / "test" -> Handler.fromResponse(Response(status = Status.Ok)), + ) + val app = route + val headRequest = Request.head("/test") + for { + response <- app.runZIO(headRequest) + } yield assertTrue( + response.status == Status.Ok, + response.body.isEmpty, + ) + }, + test("should not return 206, 304, or 416 status codes for POST requests(post_invalid_response_codes)") { + + val app = Routes( + Method.POST / "test" -> Handler.fromResponse(Response.status(Status.Ok)), + ) + + for { + res <- app.runZIO(Request.post("/test", Body.empty)) + + } yield assertTrue( + res.status != Status.PartialContent, + res.status != Status.NotModified, + res.status != Status.RequestedRangeNotSatisfiable, + res.status == Status.Ok, + ) + }, + test("should send the same headers for HEAD and GET requests (head_get_headers)") { + val getResponse = Response + .status(Status.Ok) + .addHeader(Header.ContentType(MediaType.text.html)) + .addHeader(Header.Custom("X-Custom-Header", "value")) + .copy(body = Body.fromString("
ABC
")) + + val app = Routes( + Method.GET / "test" -> Handler.fromResponse(getResponse), + Method.HEAD / "test" -> Handler.fromResponse(getResponse.copy(body = Body.empty)), + ) + + for { + getResponse <- app.runZIO(Request.get("/test")) + headResponse <- app.runZIO(Request.head("/test")) + getHeaders = getResponse.headers.toList.map(_.headerName).toSet + headHeaders = headResponse.headers.toList.map(_.headerName).toSet + } yield assertTrue( + getHeaders == headHeaders, + ) + }, + test("should reply with 501 for unknown HTTP methods (code_501_unknown_methods)") { + val app = Routes( + Method.GET / "test" -> Handler.fromResponse(Response.status(Status.Ok)), + ) + + val unknownMethodRequest = Request(method = Method.CUSTOM("ABC"), url = URL(Path.root / "test")) + + for { + response <- app.runZIO(unknownMethodRequest) + } yield assertTrue( + response.status == Status.NotImplemented, + ) + }, + test( + "should reply with 405 when the request method is not allowed for the target resource (code_405_blocked_methods)", + ) { + val app = Routes( + Method.GET / "test" -> Handler.fromResponse(Response.status(Status.Ok)), + ) + + // Testing a disallowed method (e.g., CONNECT) + val connectMethodRequest = Request(method = Method.CONNECT, url = URL(Path.root / "test")) + + for { + response <- app.runZIO(connectMethodRequest) + } yield assertTrue( + response.status == Status.MethodNotAllowed, + ) + }, + ), + suite("HTTP/1.1")( + test("should return 400 Bad Request if there is whitespace between start-line and first header field") { + val route = Method.GET / "test" -> Handler.ok + val app = Routes(route) + + val malformedRequest = + Request.get("/test").copy(headers = Headers.empty).withBody(Body.fromString("\r\nHost: localhost")) + + for { + response <- app.runZIO(malformedRequest) + } yield assertTrue(response.status == Status.BadRequest) + }, + test("should return 400 Bad Request if there is whitespace between header field and colon") { + val route = Method.GET / "test" -> Handler.ok + val app = Routes(route) + + val requestWithWhitespaceHeader = Request.get("/test").addHeader(Header.Custom("Invalid Header ", "value")) + + for { + response <- app.runZIO(requestWithWhitespaceHeader) + } yield { + assertTrue(response.status == Status.BadRequest) + } + }, + test("should not generate a bare CR in headers for HTTP/1.1(no_bare_cr)") { + val app = Routes( + Method.GET / "test" -> Handler.fromZIO { + ZIO.succeed( + Response + .status(Status.Ok) + .addHeader(Header.Custom("A", "1\r\nB: 2")), + ) + }, + ) + + val request = Request + .get("/test") + .copy(version = Version.Http_1_1) + + for { + response <- app.runZIO(request) + headersString = response.headers.toString + isValid = !headersString.contains("\r") || headersString.contains("\r\n") + } yield assertTrue(isValid) + }, + test("should allow one CRLF in front of the request line (allow_crlf_start)") { + val crlfPrefix = "\r\n".getBytes + + val validRequest = Request + .get("/valid") + .withBody(Body.fromChunk(Chunk.fromArray(crlfPrefix ++ "GET /valid HTTP/1.1".getBytes))) + + val invalidRequest = Request + .get("/invalid") + .withBody(Body.fromChunk(Chunk.fromArray(crlfPrefix ++ "GET /invalid HTTP/1.1".getBytes))) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(Response.status(Status.Ok)), + Method.GET / "invalid" -> Handler.fromResponse(Response.status(Status.NotFound)), + ) + + for { + responseValid <- app.runZIO(validRequest) + responseInvalid <- app.runZIO(invalidRequest) + } yield { + assertTrue( + responseValid.status.isSuccess || responseValid.status == Status.NotFound, + responseInvalid.status == Status.NotFound, + ) + } + }, + test("should send a 'Connection: close' option in final response (close_option_in_final_response)") { + val validRequest = Request + .get("/valid") + .addHeader(Header.Connection.Close) + + val invalidRequest = Request + .get("/invalid") + .addHeader(Header.Connection.KeepAlive) + + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.Connection.Close) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.Connection.KeepAlive) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(validRequest) + responseInvalid <- app.runZIO(invalidRequest) + } yield { + assertTrue( + responseValid.headers.toList.exists(h => + h.headerName == Header.Connection.name && h.renderedValue == "close", + ), + responseInvalid.headers.toList.exists(h => + h.headerName == Header.Connection.name && h.renderedValue == "keep-alive", + ), + ) + } + }, + ), + suite("HTTP")( + test("should return 400 Bad Request if header contains CR, LF, or NULL(reject_fields_contaning_cr_lf_nul)") { + val route = Method.GET / "test" -> Handler.ok + val app = Routes(route) + + val requestWithCRLFHeader = Request.get("/test").addHeader("InvalidHeader", "Value\r\n") + val requestWithNullHeader = Request.get("/test").addHeader("InvalidHeader", "Value\u0000") + + for { + responseCRLF <- app.runZIO(requestWithCRLFHeader) + responseNull <- app.runZIO(requestWithNullHeader) + } yield { + assertTrue(responseCRLF.status == Status.BadRequest) && + assertTrue(responseNull.status == Status.BadRequest) + } + }, + test("should send Upgrade header with 426 Upgrade Required response(send_upgrade_426)") { + val app = Routes( + Method.GET / "test" -> Handler.fromResponse( + Response + .status(Status.UpgradeRequired) + .addHeader(Header.Upgrade.Protocol("https", "1.1")), + ), + ) + + val request = Request.get("/test") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.UpgradeRequired, + response.headers.contains(Header.Upgrade.name), + ) + }, + test("should send Upgrade header with 101 Switching Protocols response(send_upgrade_101)") { + val app = Routes( + Method.GET / "switch" -> Handler.fromResponse( + Response + .status(Status.SwitchingProtocols) + .addHeader(Header.Upgrade.Protocol("https", "1.1")), + ), + ) + + val request = Request.get("/switch") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.SwitchingProtocols, + response.headers.contains(Header.Upgrade.name), + ) + }, + test("should not include Content-Length header for 1xx and 204 No Content responses(content_length_1XX_204)") { + val route1xxContinue = Method.GET / "continue" -> Handler.fromResponse(Response(status = Status.Continue)) + val route1xxSwitch = + Method.GET / "switching-protocols" -> Handler.fromResponse(Response(status = Status.SwitchingProtocols)) + val route1xxProcess = + Method.GET / "processing" -> Handler.fromResponse(Response(status = Status.Processing)) + val route204NoContent = + Method.GET / "no-content" -> Handler.fromResponse(Response(status = Status.NoContent)) + + val app = Routes(route1xxContinue, route1xxSwitch, route1xxProcess, route204NoContent) + + val requestContinue = Request.get("/continue") + val requestSwitch = Request.get("/switching-protocols") + val requestProcess = Request.get("/processing") + val requestNoContent = Request.get("/no-content") + + for { + responseContinue <- app.runZIO(requestContinue) + responseSwitch <- app.runZIO(requestSwitch) + responseProcess <- app.runZIO(requestProcess) + responseNoContent <- app.runZIO(requestNoContent) + + } yield assertTrue( + !responseContinue.headers.contains(Header.ContentLength.name), + !responseSwitch.headers.contains(Header.ContentLength.name), + !responseProcess.headers.contains(Header.ContentLength.name), + !responseNoContent.headers.contains(Header.ContentLength.name), + ) + }, + test( + "should not switch to a protocol not indicated by the client in the Upgrade header(switch_protocol_without_client)", + ) { + val app = Routes( + Method.GET / "switch" -> Handler.fromFunctionZIO { (request: Request) => + val clientUpgrade = request.headers.get(Header.Upgrade.name) + + ZIO.succeed { + clientUpgrade match { + case Some("https/1.1") => + Response + .status(Status.SwitchingProtocols) + .addHeader(Header.Upgrade.Protocol("https", "1.1")) + case Some(_) => + Response.status(Status.BadRequest) + case None => + Response.status(Status.Ok) + } + } + }, + ) + + val requestWithUpgrade = Request + .get("/switch") + .addHeader(Header.Upgrade.Protocol("https", "1.1")) + + val requestWithUnsupportedUpgrade = Request + .get("/switch") + .addHeader(Header.Upgrade.Protocol("unsupported", "1.0")) + + val requestWithoutUpgrade = Request.get("/switch") + + for { + responseWithUpgrade <- app.runZIO(requestWithUpgrade) + responseWithUnsupportedUpgrade <- app.runZIO(requestWithUnsupportedUpgrade) + responseWithoutUpgrade <- app.runZIO(requestWithoutUpgrade) + + } yield assertTrue( + responseWithUpgrade.status == Status.SwitchingProtocols, + responseWithUpgrade.headers.contains(Header.Upgrade.name), + responseWithUnsupportedUpgrade.status == Status.BadRequest, + responseWithoutUpgrade.status == Status.Ok, + ) + }, + test( + "should send 100 Continue before 101 Switching Protocols when both Upgrade and Expect headers are present(continue_before_upgrade)", + ) { + val continueHandler = Handler.fromZIO { + ZIO.succeed(Response.status(Status.Continue)) + } + + val switchingProtocolsHandler = Handler.fromZIO { + ZIO.succeed( + Response + .status(Status.SwitchingProtocols) + .addHeader(Header.Connection.KeepAlive) + .addHeader(Header.Upgrade.Protocol("https", "1.1")), + ) + } + val app = Routes( + Method.POST / "upgrade" -> continueHandler, + Method.GET / "switch" -> switchingProtocolsHandler, + ) + val initialRequest = Request + .post("/upgrade", Body.empty) + .addHeader(Header.Expect.`100-continue`) + .addHeader(Header.Connection.KeepAlive) + .addHeader(Header.Upgrade.Protocol("https", "1.1")) + + val followUpRequest = Request.get("/switch") + + for { + firstResponse <- app.runZIO(initialRequest) + secondResponse <- app.runZIO(followUpRequest) + + } yield assertTrue( + firstResponse.status == Status.Continue, + secondResponse.status == Status.SwitchingProtocols, + secondResponse.headers.contains(Header.Upgrade.name), + secondResponse.headers.contains(Header.Connection.name), + ) + }, + test("should not return forbidden duplicate headers in response(duplicate_fields)") { + val app = Routes( + Method.GET / "test" -> Handler.fromResponse( + Response + .status(Status.Ok) + .addHeader(Header.XFrameOptions.Deny) + .addHeader(Header.XFrameOptions.SameOrigin), + ), + ) + for { + response <- app.runZIO(Request.get("/test")) + } yield { + val xFrameOptionsHeaders = response.headers.toList.collect { + case h if h.headerName == Header.XFrameOptions.name => h + } + assertTrue(xFrameOptionsHeaders.length == 1) + } + }, + suite("Content-Length")( + test("Content-Length in HEAD must match the one in GET (content_length_same_head_get)") { + val getResponse = Response + .status(Status.Ok) + .addHeader(Header.ContentLength(14)) + .copy(body = Body.fromString("
ABC
")) + + val app = Routes( + Method.GET / "test" -> Handler.fromResponse(getResponse), + Method.HEAD / "test" -> Handler.fromResponse(getResponse.copy(body = Body.empty)), + ) + + for { + getResponse <- app.runZIO(Request.get("/test")) + headResponse <- app.runZIO(Request.head("/test")) + getContentLength = getResponse.headers.get(Header.ContentLength.name).map(_.toInt) + headContentLength = headResponse.headers.get(Header.ContentLength.name).map(_.toInt) + } yield assertTrue( + headContentLength == getContentLength, + ) + }, + test("Content-Length in 304 Not Modified must match the one in 200 OK (content_length_same_304_200)") { + val app = Routes( + Method.GET / "test" -> Handler.fromFunction { (request: Request) => + request.headers.get(Header.IfModifiedSince.name) match { + case Some(_) => + Response.status(Status.NotModified).addHeader(Header.ContentLength(14)).copy(body = Body.empty) + case None => + Response + .status(Status.Ok) + .addHeader(Header.ContentLength(14)) + .copy(body = Body.fromString("
ABC
")) + } + }, + ) + + val conditionalRequest = Request + .get("/test") + .addHeader( + Header.IfModifiedSince( + ZonedDateTime.parse("Thu, 20 Mar 2025 07:28:00 GMT", DateTimeFormatter.RFC_1123_DATE_TIME), + ), + ) + + for { + normalResponse <- app.runZIO(Request.get("/test")) + conditionalResponse <- app.runZIO(conditionalRequest) + normalContentLength = normalResponse.headers.get(Header.ContentLength.name).map(_.toInt) + conditionalContentLength = conditionalResponse.headers.get(Header.ContentLength.name).map(_.toInt) + } yield assertTrue( + normalContentLength == conditionalContentLength, + ) + }, + ), + ), + suite("cache-control")( + test("Cache-Control should not have quoted string for max-age directive(response_directive_max_age)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.CacheControl.MaxAge(5)) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Cache-Control", """max-age="5"""")) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.headers.get(Header.CacheControl.name).contains("max-age=5"), + responseInvalid.headers.get(Header.CacheControl.name).contains("""max-age="5""""), + ) + }, + test("Cache-Control should not have quoted string for s-maxage directive(response_directive_s_maxage)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.CacheControl.SMaxAge(10)) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Cache-Control", """s-maxage="10"""")) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.headers.get(Header.CacheControl.name).contains("s-maxage=10"), + responseInvalid.headers.get(Header.CacheControl.name).contains("""s-maxage="10""""), + ) + }, + test("Cache-Control should use quoted-string form for no-cache directive(response_directive_no_cache)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Cache-Control", """no-cache="age"""")) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Cache-Control", "no-cache=age")) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.headers.get(Header.CacheControl.name).contains("""no-cache="age""""), + responseInvalid.headers.get(Header.CacheControl.name).contains("no-cache=age"), + ) + }, + test("Cache-Control should use quoted-string form for private directive(response_directive_private)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Cache-Control", """private="x-frame-options"""")) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Cache-Control", "private=x-frame-options")) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.headers.get(Header.CacheControl.name).contains("""private="x-frame-options""""), + responseInvalid.headers.get(Header.CacheControl.name).contains("private=x-frame-options"), + ) + }, + ), + suite("cookies")( + test("should not have duplicate cookie attributes in Set-Cookie header(duplicate_cookie_attribute)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.SetCookie(Cookie.Response("test", "test", path = Some(Path.root)))) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Set-Cookie", "test=test; path=/; path=/abc")) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield { + val validCookieAttributes = responseValid.headers.toList.collect { + case h if h.headerName == Header.SetCookie.name => h.renderedValue + } + val invalidCookieAttributes = responseInvalid.headers.toList.collect { + case h if h.headerName == "Set-Cookie" => h.renderedValue + } + assertTrue( + validCookieAttributes.nonEmpty, + validCookieAttributes.exists(_.toLowerCase.contains("path=/")), + !validCookieAttributes.exists(_.toLowerCase.contains("path=/abc")), + ) && + assertTrue( + invalidCookieAttributes.exists(_.contains("path=/")), + invalidCookieAttributes.exists(_.contains("path=/abc")), + ) + } + }, + test("should not have duplicate cookies with the same name(duplicate_cookies)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.SetCookie(Cookie.Response("test", "test"))) + .addHeader(Header.SetCookie(Cookie.Response("test2", "test2"))) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.SetCookie(Cookie.Response("test", "test"))) + .addHeader(Header.SetCookie(Cookie.Response("test", "test2"))) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield { + val validCookies = responseValid.headers.toList.collect { + case h if h.headerName == Header.SetCookie.name => h.renderedValue + } + val invalidCookies = responseInvalid.headers.toList.collect { + case h if h.headerName == Header.SetCookie.name => h.renderedValue + } + assertTrue( + validCookies.count(_.contains("test=")) == 1, + ) && + assertTrue( + invalidCookies.count(_.contains("test=")) == 2, + ) + } + }, + test("should use IMF-fixdate for cookie expiration date(cookie_IMF_fixdate)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.SetCookie(Cookie.Response("test", "test", maxAge = Some(Duration.fromSeconds(86400))))) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Set-Cookie", "test=test; expires=Thu, 20 Mar 25 15:14:45 GMT")) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield { + val expiresValid = responseValid.headers.toList.exists(_.renderedValue.contains("Expires=")) + val expiresInvalid = + responseInvalid.headers.toList.exists(_.renderedValue.contains("expires=Thu, 20 Mar 25")) + + assertTrue( + expiresValid, + expiresInvalid, + ) + } + }, + ), + suite("conformance")( + test("should not include Content-Length header for 204 No Content responses") { + val route = Method.GET / "no-content" -> Handler.fromResponse(Response(status = Status.NoContent)) + val app = Routes(route) + + val request = Request.get("/no-content") + for { + response <- app.runZIO(request) + } yield assertTrue(!response.headers.contains(Header.ContentLength.name)) + }, + test("should not send content for 304 Not Modified responses") { + val app = Routes( + Method.GET / "not-modified" -> Handler.fromResponse( + Response.status(Status.NotModified), + ), + ) + + val request = Request.get("/not-modified") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.NotModified, + response.body.isEmpty, + !response.headers.contains(Header.ContentLength.name), + !response.headers.contains(Header.TransferEncoding.name), + ) + }, + ), + ) +} From 969afa94fca8f3df5c350b2304532baf650edd24 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:38:09 +0530 Subject: [PATCH 02/12] workflow for conformance suite added --- .github/workflows/http-conformance.yml | 73 ++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 .github/workflows/http-conformance.yml diff --git a/.github/workflows/http-conformance.yml b/.github/workflows/http-conformance.yml new file mode 100644 index 0000000000..b4a5fc90ea --- /dev/null +++ b/.github/workflows/http-conformance.yml @@ -0,0 +1,73 @@ +name: HTTP Spec Conformance Test + +on: + pull_request: + branches: ["**"] + push: + branches: ["**"] + tags: [v*] + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + JDK_JAVA_OPTIONS: "-Xms4G -Xmx8G -XX:+UseG1GC -Xss10M -XX:ReservedCodeCacheSize=1G -XX:NonProfiledCodeHeapSize=512m -Dfile.encoding=UTF-8" + SBT_OPTS: "-Xms4G -Xmx8G -XX:+UseG1GC -Xss10M -XX:ReservedCodeCacheSize=1G -XX:NonProfiledCodeHeapSize=512m -Dfile.encoding=UTF-8" + +jobs: + build: + name: Build and Test + strategy: + matrix: + os: [ubuntu-latest] + scala: [2.12.19, 2.13.14, 3.3.3] + java: + - graal_graalvm@17 + - graal_graalvm@21 + - temurin@17 + - temurin@21 + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + + steps: + - name: Checkout current branch (full) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup GraalVM (graal_graalvm@17) + if: matrix.java == 'graal_graalvm@17' + uses: graalvm/setup-graalvm@v1 + with: + java-version: 17 + distribution: graalvm + components: native-image + github-token: ${{ secrets.GITHUB_TOKEN }} + cache: sbt + + - name: Setup GraalVM (graal_graalvm@21) + if: matrix.java == 'graal_graalvm@21' + uses: graalvm/setup-graalvm@v1 + with: + java-version: 21 + distribution: graalvm + components: native-image + github-token: ${{ secrets.GITHUB_TOKEN }} + cache: sbt + + - name: Setup Java (temurin@17) + if: matrix.java == 'temurin@17' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + cache: sbt + + - name: Setup Java (temurin@21) + if: matrix.java == 'temurin@21' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + cache: sbt + + - name: Run HTTP Conformance Tests + run: sbt "project zioHttpJVM" "testOnly zio.http.ConformanceSpec" From 698dd7c42fc68cdad2ed81857d67ba6c108377d6 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Mon, 30 Sep 2024 20:36:16 +0530 Subject: [PATCH 03/12] fix(conformance): forbidden duplicate headers --- .../src/main/scala/zio/http/internal/HeaderModifier.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zio-http/shared/src/main/scala/zio/http/internal/HeaderModifier.scala b/zio-http/shared/src/main/scala/zio/http/internal/HeaderModifier.scala index 255358d8c5..ea2ba32b05 100644 --- a/zio-http/shared/src/main/scala/zio/http/internal/HeaderModifier.scala +++ b/zio-http/shared/src/main/scala/zio/http/internal/HeaderModifier.scala @@ -32,7 +32,11 @@ import zio.http._ */ trait HeaderModifier[+A] { self => final def addHeader(header: Header): A = - addHeaders(Headers(header)) + if (header.headerName == Header.XFrameOptions.name) { + updateHeaders(headers => Headers(headers.filterNot(_.headerName == Header.XFrameOptions.name)) ++ Headers(header)) + } else { + addHeaders(Headers(header)) + } final def addHeader(name: CharSequence, value: CharSequence): A = addHeaders(Headers.apply(name, value)) From bb47958f934a1c4d9b7d6ed3524b028cb208f379 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 01:31:05 +0530 Subject: [PATCH 04/12] fix(conformance): tests and move to other suite for e2e validation --- .github/workflows/http-conformance.yml | 4 +- .../netty/server/ServerInboundHandler.scala | 44 +++++++- .../scala/zio/http/ConformanceE2ESpec.scala | 103 ++++++++++++++++++ .../test/scala/zio/http/ConformanceSpec.scala | 94 ++-------------- 4 files changed, 156 insertions(+), 89 deletions(-) create mode 100644 zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala diff --git a/.github/workflows/http-conformance.yml b/.github/workflows/http-conformance.yml index b4a5fc90ea..e99c42f029 100644 --- a/.github/workflows/http-conformance.yml +++ b/.github/workflows/http-conformance.yml @@ -1,4 +1,4 @@ -name: HTTP Spec Conformance Test +name: HTTP Conformance on: pull_request: @@ -70,4 +70,4 @@ jobs: cache: sbt - name: Run HTTP Conformance Tests - run: sbt "project zioHttpJVM" "testOnly zio.http.ConformanceSpec" + run: sbt "project zioHttpJVM" "testOnly zio.http.ConformanceSpec zio.http.ConformanceE2ESpec" diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala index 74340d825e..a7f9d0ba91 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala @@ -87,12 +87,19 @@ private[zio] final case class ServerInboundHandler( ) releaseRequest() } else { - val req = makeZioRequest(ctx, jReq) - val exit = handler(req) - if (attemptImmediateWrite(ctx, req.method, exit)) { + val req = makeZioRequest(ctx, jReq) + if (!validateHostHeader(req)) { + attemptFastWrite(ctx, req.method, Response.status(Status.BadRequest)) releaseRequest() } else { - writeResponse(ctx, runtime, exit, req)(releaseRequest) + + val exit = handler(req) + if (attemptImmediateWrite(ctx, req.method, exit)) { + releaseRequest() + } else { + writeResponse(ctx, runtime, exit, req)(releaseRequest) + + } } } } finally { @@ -108,6 +115,34 @@ private[zio] final case class ServerInboundHandler( } + private def validateHostHeader(req: Request): Boolean = { + req.headers.get("Host") match { + case Some(host) => + val parts = host.split(":") + val hostname = parts(0) + val isValidHost = validateHostname(hostname) + val isValidPort = parts.length == 1 || (parts.length == 2 && parts(1).forall(_.isDigit)) + val isValid = isValidHost && isValidPort + println(s"Host: $host, isValidHost: $isValidHost, isValidPort: $isValidPort, isValid: $isValid") + isValid + case None => + println("Host header missing!") + false + } + } + +// Validate a regular hostname (based on RFC 1035) + private def validateHostname(hostname: String): Boolean = { + if (hostname.isEmpty || hostname.contains("_")) { + return false + } + val labels = hostname.split("\\.") + if (labels.exists(label => label.isEmpty || label.length > 63 || label.startsWith("-") || label.endsWith("-"))) { + return false + } + hostname.forall(c => c.isLetterOrDigit || c == '.' || c == '-') && hostname.length <= 253 + } + override def exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable): Unit = cause match { case ioe: IOException if { @@ -262,7 +297,6 @@ private[zio] final case class ServerInboundHandler( remoteCertificate = clientCert, ) } - } /* diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala new file mode 100644 index 0000000000..a6a61df9d1 --- /dev/null +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala @@ -0,0 +1,103 @@ +package zio.http + +import zio._ +import zio.test.Assertion._ +import zio.test.TestAspect._ +import zio.test._ + +import zio.http._ +import zio.http.internal.{DynamicServer, RoutesRunnableSpec} +import zio.http.netty.NettyConfig + +object ConformanceE2ESpec extends RoutesRunnableSpec { + + private val port = 8080 + private val MaxSize = 1024 * 10 + val configApp = Server.Config.default + .requestDecompression(true) + .disableRequestStreaming(MaxSize) + .port(port) + .responseCompression() + + private val app = serve + + def conformanceSpec = suite("ConformanceE2ESpec")( + test("should return 400 Bad Request if Host header is missing") { + val routes = Handler.ok.toRoutes + + val res = routes.deploy.status.run(path = Path.root, headers = Headers(Header.Host("%%%%invalid%%%%"))) + assertZIO(res)(equalTo(Status.BadRequest)) + }, + test("should return 200 OK if Host header is present") { + val routes = Handler.ok.toRoutes + + val res = routes.deploy.status.run(path = Path.root, headers = Headers(Header.Host("localhost"))) + assertZIO(res)(equalTo(Status.Ok)) + }, + test("should reply with 501 for unknown HTTP methods (code_501_unknown_methods)") { + val routes = Handler.ok.toRoutes + + val res = routes.deploy.status.run(path = Path.root, method = Method.CUSTOM("ABC")) + + assertZIO(res)(equalTo(Status.NotImplemented)) + }, + test( + "should reply with 405 when the request method is not allowed for the target resource (code_405_blocked_methods)", + ) { + val routes = Handler.ok.toRoutes + + val res = routes.deploy.status.run(path = Path.root, method = Method.CONNECT) + assertZIO(res)(equalTo(Status.MethodNotAllowed)) + }, + test("should return 400 Bad Request if header contains CR, LF, or NULL (reject_fields_containing_cr_lf_nul)") { + val routes = Handler.ok.toRoutes + + val resCRLF = + routes.deploy.status.run(path = Path.root / "test", headers = Headers("InvalidHeader" -> "Value\r\n")) + val resNull = + routes.deploy.status.run(path = Path.root / "test", headers = Headers("InvalidHeader" -> "Value\u0000")) + + for { + responseCRLF <- resCRLF + responseNull <- resNull + } yield assertTrue( + responseCRLF == Status.BadRequest, + responseNull == Status.BadRequest, + ) + }, + test("should return 400 Bad Request if there is whitespace between start-line and first header field") { + val route = Method.GET / "test" -> Handler.ok + val routes = Routes(route) + + val malformedRequest = Request + .get("/test") + .copy(headers = Headers.empty) + .withBody(Body.fromString("\r\nHost: localhost")) + + val res = routes.deploy.status.run(path = Path.root / "test", headers = malformedRequest.headers) + assertZIO(res)(equalTo(Status.BadRequest)) + }, + test("should return 400 Bad Request if there is whitespace between header field and colon") { + val route = Method.GET / "test" -> Handler.ok + val routes = Routes(route) + + val requestWithWhitespaceHeader = Request.get("/test").addHeader(Header.Custom("Invalid Header ", "value")) + + val res = routes.deploy.status.run(path = Path.root / "test", headers = requestWithWhitespaceHeader.headers) + assertZIO(res)(equalTo(Status.BadRequest)) + }, + ) + + override def spec = + suite("ConformanceE2ESpec") { + val spec = conformanceSpec + suite("app without request streaming") { app.as(List(spec)) } + }.provideShared( + DynamicServer.live, + ZLayer.succeed(configApp), + Server.customized, + Client.default, + ZLayer.succeed(NettyConfig.default), + ) @@ sequential @@ withLiveClock + +} diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala index e12913b7cf..9b448ab397 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala @@ -21,7 +21,6 @@ object ConformanceSpec extends ZIOSpecDefault { * * Paper URL: https://doi.org/10.1145/3634737.3637678 * GitHub Project: https://github.com/cispa/http-conformance - * */ val validUrl = URL.decode("http://example.com").toOption.getOrElse(URL.root) @@ -504,48 +503,6 @@ object ConformanceSpec extends ZIOSpecDefault { }, ), suite("HTTP Headers")( - suite("code_400_after_bad_host_request")( - test("should return 200 OK if Host header is present") { - val route = Method.GET / "test" -> Handler.ok - val app = Routes(route) - val requestWithHost = Request.get("/test").addHeader(Header.Host("localhost")) - for { - response <- app.runZIO(requestWithHost) - } yield assertTrue(response.status == Status.Ok) - }, - test("should return 400 Bad Request if Host header is missing") { - val route = Method.GET / "test" -> Handler.ok - val app = Routes(route) - val requestWithoutHost = Request.get("/test") - - for { - response <- app.runZIO(requestWithoutHost) - } yield assertTrue(response.status == Status.BadRequest) - }, - test("should return 400 Bad Request if there are multiple Host headers") { - val route = Method.GET / "test" -> Handler.ok - val app = Routes(route) - val requestWithTwoHosts = Request - .get("/test") - .addHeader(Header.Host("example.com")) - .addHeader(Header.Host("another.com")) - - for { - response <- app.runZIO(requestWithTwoHosts) - } yield assertTrue(response.status == Status.BadRequest) - }, - test("should return 400 Bad Request if Host header is invalid") { - val route = Method.GET / "test" -> Handler.ok - val app = Routes(route) - val requestWithInvalidHost = Request - .get("/test") - .addHeader(Header.Host("invalid_host")) - - for { - response <- app.runZIO(requestWithInvalidHost) - } yield assertTrue(response.status == Status.BadRequest) - }, - ), test("should not include Content-Length header for 2XX CONNECT responses(content_length_2XX_connect)") { val app = Routes( Method.CONNECT / "" -> Handler.fromResponse( @@ -764,22 +721,6 @@ object ConformanceSpec extends ZIOSpecDefault { }, ), suite("transfer_encoding_http11")( - test("should not send Transfer-Encoding in response if request HTTP version is below 1.1") { - val app = Routes( - Method.GET / "test" -> Handler.fromResponse( - Response.ok.addHeader(Header.TransferEncoding.Chunked), - ), - ) - - val request = Request.get("/test").copy(version = Version.`HTTP/1.0`) - - for { - response <- app.runZIO(request) - } yield assertTrue( - response.status == Status.Ok, - !response.headers.contains(Header.TransferEncoding.name), - ) - }, test("should send Transfer-Encoding in response if request HTTP version is 1.1 or higher") { val app = Routes( Method.GET / "test" -> Handler.fromResponse( @@ -850,6 +791,18 @@ object ConformanceSpec extends ZIOSpecDefault { getHeaders == headHeaders, ) }, + test("404 response for truly non-existent path") { + val app = Routes( + Method.GET / "existing-path" -> Handler.ok, + ) + val request = Request.get(URL(Path.root / "non-existent-path")) + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.NotFound, + ) + }, test("should reply with 501 for unknown HTTP methods (code_501_unknown_methods)") { val app = Routes( Method.GET / "test" -> Handler.fromResponse(Response.status(Status.Ok)), @@ -881,29 +834,6 @@ object ConformanceSpec extends ZIOSpecDefault { }, ), suite("HTTP/1.1")( - test("should return 400 Bad Request if there is whitespace between start-line and first header field") { - val route = Method.GET / "test" -> Handler.ok - val app = Routes(route) - - val malformedRequest = - Request.get("/test").copy(headers = Headers.empty).withBody(Body.fromString("\r\nHost: localhost")) - - for { - response <- app.runZIO(malformedRequest) - } yield assertTrue(response.status == Status.BadRequest) - }, - test("should return 400 Bad Request if there is whitespace between header field and colon") { - val route = Method.GET / "test" -> Handler.ok - val app = Routes(route) - - val requestWithWhitespaceHeader = Request.get("/test").addHeader(Header.Custom("Invalid Header ", "value")) - - for { - response <- app.runZIO(requestWithWhitespaceHeader) - } yield { - assertTrue(response.status == Status.BadRequest) - } - }, test("should not generate a bare CR in headers for HTTP/1.1(no_bare_cr)") { val app = Routes( Method.GET / "test" -> Handler.fromZIO { From 453c609b054fb9501ba08a1f4051b6617d5722e9 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 01:48:55 +0530 Subject: [PATCH 05/12] chore: cleanup and fix 404 and 405 tests --- .../main/scala/zio/http/RoutePattern.scala | 6 +++ .../src/main/scala/zio/http/Routes.scala | 48 ++++++++++++------- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala b/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala index c42739408b..7bcdff3505 100644 --- a/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala +++ b/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala @@ -185,6 +185,12 @@ object RoutePattern { else forMethod ++ wildcardsTree.get(path) } + def getAllMethods(path: Path): Set[Method] = { + roots.collect { + case (method, subtree) if subtree.get(path).nonEmpty => method + }.toSet + } + def map[B](f: A => B): Tree[B] = Tree(roots.map { case (k, v) => k -> v.map(f) diff --git a/zio-http/shared/src/main/scala/zio/http/Routes.scala b/zio-http/shared/src/main/scala/zio/http/Routes.scala index 5847d9524e..05503d5d0d 100644 --- a/zio-http/shared/src/main/scala/zio/http/Routes.scala +++ b/zio-http/shared/src/main/scala/zio/http/Routes.scala @@ -248,22 +248,34 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s val tree = self.tree Handler .fromFunctionHandler[Request] { req => - val chunk = tree.get(req.method, req.path) - chunk.length match { - case 0 => Handler.notFound - case 1 => chunk(0) - case n => // TODO: Support precomputed fallback among all chunk elements - var acc = chunk(0) - var i = 1 - while (i < n) { - val h = chunk(i) - acc = acc.catchAll { response => - if (response.status == Status.NotFound) h - else Handler.fail(response) - } - i += 1 + val chunk = tree.get(req.method, req.path) + val allowedMethods = tree.getAllMethods(req.path) + + req.method match { + case Method.CUSTOM(_) => + Handler.fromZIO(ZIO.succeed(Response.status(Status.NotImplemented))) + case _ if chunk.isEmpty && allowedMethods.nonEmpty => + Handler.fromZIO(ZIO.succeed(Response.status(Status.MethodNotAllowed))) + + case _ if chunk.isEmpty && allowedMethods.isEmpty => + Handler.notFound + case _ => + chunk.length match { + case 0 => Handler.notFound + case 1 => chunk(0) + case n => // TODO: Support precomputed fallback among all chunk elements + var acc = chunk(0) + var i = 1 + while (i < n) { + val h = chunk(i) + acc = acc.catchAll { response => + if (response.status == Status.NotFound) h + else Handler.fail(response) + } + i += 1 + } + acc } - acc } } .merge @@ -287,6 +299,7 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s } _tree.asInstanceOf[Routes.Tree[Env]] } + } object Routes extends RoutesCompanionVersionSpecific { @@ -344,6 +357,9 @@ object Routes extends RoutesCompanionVersionSpecific { empty @@ Middleware.serveResources(path, resourcePrefix) private[http] final case class Tree[-Env](tree: RoutePattern.Tree[RequestHandler[Env, Response]]) { self => + + def getAllMethods(path: Path): Set[Method] = tree.getAllMethods(path) + final def ++[Env1 <: Env](that: Tree[Env1]): Tree[Env1] = Tree(self.tree ++ that.tree) @@ -357,7 +373,7 @@ object Routes extends RoutesCompanionVersionSpecific { final def get(method: Method, path: Path): Chunk[RequestHandler[Env, Response]] = tree.get(method, path) } - private[http] object Tree { + private[http] object Tree { val empty: Tree[Any] = Tree(RoutePattern.Tree.empty) def fromRoutes[Env](routes: Chunk[zio.http.Route[Env, Response]])(implicit trace: Trace): Tree[Env] = From 73a209b2e67a170a045b7df8ad3c32714f506caf Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 01:56:02 +0530 Subject: [PATCH 06/12] Update ServerInboundHandler.scala --- .../main/scala/zio/http/netty/server/ServerInboundHandler.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala index a7f9d0ba91..d87261e25b 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala @@ -123,10 +123,8 @@ private[zio] final case class ServerInboundHandler( val isValidHost = validateHostname(hostname) val isValidPort = parts.length == 1 || (parts.length == 2 && parts(1).forall(_.isDigit)) val isValid = isValidHost && isValidPort - println(s"Host: $host, isValidHost: $isValidHost, isValidPort: $isValidPort, isValid: $isValid") isValid case None => - println("Host header missing!") false } } From 1948dc44b6513dcb59e0fb8df00ee8acb0c0bb13 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 20:58:57 +0530 Subject: [PATCH 07/12] feat(conformance): 404 --- .../zio/http/netty/model/Conversions.scala | 8 ++++++++ .../scala/zio/http/ConformanceE2ESpec.scala | 15 -------------- .../test/scala/zio/http/ConformanceSpec.scala | 19 ++---------------- .../src/main/scala/zio/http/Handler.scala | 12 +++++++++++ .../src/main/scala/zio/http/Routes.scala | 20 +++++++++---------- 5 files changed, 32 insertions(+), 42 deletions(-) diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/model/Conversions.scala b/zio-http/jvm/src/main/scala/zio/http/netty/model/Conversions.scala index 374c9ba27f..a67cacc879 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/model/Conversions.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/model/Conversions.scala @@ -71,6 +71,10 @@ private[netty] object Conversions { url0.relative.addLeadingSlash.encode } + private def validateHeaderValue(value: String): Boolean = { + value.contains('\r') || value.contains('\n') || value.contains('\u0000') + } + private def nettyHeadersIterator(headers: HttpHeaders): Iterator[Header] = new AbstractIterator[Header] { private val nettyIterator = headers.iteratorCharSequence() @@ -99,6 +103,10 @@ private[netty] object Conversions { while (iter.hasNext) { val header = iter.next() val name = header.headerName + val value = header.renderedValueAsCharSequence.toString + if (validateHeaderValue(value)) { + throw new IllegalArgumentException(s"Invalid header value containing prohibited characters in header $name") + } if (name == setCookieName) { nettyHeaders.add(name, header.renderedValueAsCharSequence) } else { diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala index a6a61df9d1..6356d30769 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala @@ -34,21 +34,6 @@ object ConformanceE2ESpec extends RoutesRunnableSpec { val res = routes.deploy.status.run(path = Path.root, headers = Headers(Header.Host("localhost"))) assertZIO(res)(equalTo(Status.Ok)) }, - test("should reply with 501 for unknown HTTP methods (code_501_unknown_methods)") { - val routes = Handler.ok.toRoutes - - val res = routes.deploy.status.run(path = Path.root, method = Method.CUSTOM("ABC")) - - assertZIO(res)(equalTo(Status.NotImplemented)) - }, - test( - "should reply with 405 when the request method is not allowed for the target resource (code_405_blocked_methods)", - ) { - val routes = Handler.ok.toRoutes - - val res = routes.deploy.status.run(path = Path.root, method = Method.CONNECT) - assertZIO(res)(equalTo(Status.MethodNotAllowed)) - }, test("should return 400 Bad Request if header contains CR, LF, or NULL (reject_fields_containing_cr_lf_nul)") { val routes = Handler.ok.toRoutes diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala index 9b448ab397..7a83f6d5e8 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala @@ -21,6 +21,7 @@ object ConformanceSpec extends ZIOSpecDefault { * * Paper URL: https://doi.org/10.1145/3634737.3637678 * GitHub Project: https://github.com/cispa/http-conformance + * */ val validUrl = URL.decode("http://example.com").toOption.getOrElse(URL.root) @@ -759,7 +760,6 @@ object ConformanceSpec extends ZIOSpecDefault { val app = Routes( Method.POST / "test" -> Handler.fromResponse(Response.status(Status.Ok)), ) - for { res <- app.runZIO(Request.post("/test", Body.empty)) @@ -791,7 +791,7 @@ object ConformanceSpec extends ZIOSpecDefault { getHeaders == headHeaders, ) }, - test("404 response for truly non-existent path") { + test("should reply with 404 response for truly non-existent path") { val app = Routes( Method.GET / "existing-path" -> Handler.ok, ) @@ -919,21 +919,6 @@ object ConformanceSpec extends ZIOSpecDefault { }, ), suite("HTTP")( - test("should return 400 Bad Request if header contains CR, LF, or NULL(reject_fields_contaning_cr_lf_nul)") { - val route = Method.GET / "test" -> Handler.ok - val app = Routes(route) - - val requestWithCRLFHeader = Request.get("/test").addHeader("InvalidHeader", "Value\r\n") - val requestWithNullHeader = Request.get("/test").addHeader("InvalidHeader", "Value\u0000") - - for { - responseCRLF <- app.runZIO(requestWithCRLFHeader) - responseNull <- app.runZIO(requestWithNullHeader) - } yield { - assertTrue(responseCRLF.status == Status.BadRequest) && - assertTrue(responseNull.status == Status.BadRequest) - } - }, test("should send Upgrade header with 426 Upgrade Required response(send_upgrade_426)") { val app = Routes( Method.GET / "test" -> Handler.fromResponse( diff --git a/zio-http/shared/src/main/scala/zio/http/Handler.scala b/zio-http/shared/src/main/scala/zio/http/Handler.scala index 2bbdae0034..8cdcb369c9 100644 --- a/zio-http/shared/src/main/scala/zio/http/Handler.scala +++ b/zio-http/shared/src/main/scala/zio/http/Handler.scala @@ -1018,6 +1018,18 @@ object Handler extends HandlerPlatformSpecific with HandlerVersionSpecific { def notFound(message: => String): Handler[Any, Nothing, Any, Response] = error(Status.NotFound, message) + /** + * Creates a handler which always responds with a 501 status code. + */ + def notImplemented: Handler[Any, Nothing, Any, Response] = + error(Status.NotImplemented) + + /** + * Creates a handler which always responds with a 501 status code. + */ + def notImplemented(message: => String): Handler[Any, Nothing, Any, Response] = + error(Status.NotImplemented, message) + /** * Creates a handler which always responds with a 200 status code. */ diff --git a/zio-http/shared/src/main/scala/zio/http/Routes.scala b/zio-http/shared/src/main/scala/zio/http/Routes.scala index 05503d5d0d..642044ca0e 100644 --- a/zio-http/shared/src/main/scala/zio/http/Routes.scala +++ b/zio-http/shared/src/main/scala/zio/http/Routes.scala @@ -250,18 +250,18 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s .fromFunctionHandler[Request] { req => val chunk = tree.get(req.method, req.path) val allowedMethods = tree.getAllMethods(req.path) - req.method match { - case Method.CUSTOM(_) => - Handler.fromZIO(ZIO.succeed(Response.status(Status.NotImplemented))) - case _ if chunk.isEmpty && allowedMethods.nonEmpty => - Handler.fromZIO(ZIO.succeed(Response.status(Status.MethodNotAllowed))) - - case _ if chunk.isEmpty && allowedMethods.isEmpty => - Handler.notFound - case _ => + case Method.CUSTOM(_) => + Handler.notImplemented + case _ => chunk.length match { - case 0 => Handler.notFound + case 0 => + if (allowedMethods.nonEmpty) { + val allowHeader = Header.Allow(NonEmptyChunk.fromIterableOption(allowedMethods).get) + Handler.methodNotAllowed.addHeader(allowHeader) + } else { + Handler.notFound + } case 1 => chunk(0) case n => // TODO: Support precomputed fallback among all chunk elements var acc = chunk(0) From 6514ff7adcb170fa7e153c08f6e91135a4b4aa9a Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 22:57:29 +0530 Subject: [PATCH 08/12] chore(conformance): cleanup netty exception tests --- .../scala/zio/http/ConformanceE2ESpec.scala | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala index 6356d30769..b295dcb9e1 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala @@ -34,43 +34,6 @@ object ConformanceE2ESpec extends RoutesRunnableSpec { val res = routes.deploy.status.run(path = Path.root, headers = Headers(Header.Host("localhost"))) assertZIO(res)(equalTo(Status.Ok)) }, - test("should return 400 Bad Request if header contains CR, LF, or NULL (reject_fields_containing_cr_lf_nul)") { - val routes = Handler.ok.toRoutes - - val resCRLF = - routes.deploy.status.run(path = Path.root / "test", headers = Headers("InvalidHeader" -> "Value\r\n")) - val resNull = - routes.deploy.status.run(path = Path.root / "test", headers = Headers("InvalidHeader" -> "Value\u0000")) - - for { - responseCRLF <- resCRLF - responseNull <- resNull - } yield assertTrue( - responseCRLF == Status.BadRequest, - responseNull == Status.BadRequest, - ) - }, - test("should return 400 Bad Request if there is whitespace between start-line and first header field") { - val route = Method.GET / "test" -> Handler.ok - val routes = Routes(route) - - val malformedRequest = Request - .get("/test") - .copy(headers = Headers.empty) - .withBody(Body.fromString("\r\nHost: localhost")) - - val res = routes.deploy.status.run(path = Path.root / "test", headers = malformedRequest.headers) - assertZIO(res)(equalTo(Status.BadRequest)) - }, - test("should return 400 Bad Request if there is whitespace between header field and colon") { - val route = Method.GET / "test" -> Handler.ok - val routes = Routes(route) - - val requestWithWhitespaceHeader = Request.get("/test").addHeader(Header.Custom("Invalid Header ", "value")) - - val res = routes.deploy.status.run(path = Path.root / "test", headers = requestWithWhitespaceHeader.headers) - assertZIO(res)(equalTo(Status.BadRequest)) - }, ) override def spec = From 943167c83099c8d4608376eca404c2848271a344 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 23:03:37 +0530 Subject: [PATCH 09/12] fix(conformance): fmt --- zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala index 7a83f6d5e8..fa80d17dd9 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala @@ -19,9 +19,8 @@ object ConformanceSpec extends ZIOSpecDefault { * Stock, presented at the 19th ACM Asia Conference on Computer and * Communications Security (ASIA CCS) 2024. * - * Paper URL: https://doi.org/10.1145/3634737.3637678 - * GitHub Project: https://github.com/cispa/http-conformance - * + * Paper URL: https://doi.org/10.1145/3634737.3637678 GitHub Project: + * https://github.com/cispa/http-conformance */ val validUrl = URL.decode("http://example.com").toOption.getOrElse(URL.root) From 5257f23a36e83ba196889ba6fa23472aceba1e4f Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 23:07:13 +0530 Subject: [PATCH 10/12] remove netty exceptions added --- .../src/main/scala/zio/http/netty/model/Conversions.scala | 8 -------- 1 file changed, 8 deletions(-) diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/model/Conversions.scala b/zio-http/jvm/src/main/scala/zio/http/netty/model/Conversions.scala index a67cacc879..374c9ba27f 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/model/Conversions.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/model/Conversions.scala @@ -71,10 +71,6 @@ private[netty] object Conversions { url0.relative.addLeadingSlash.encode } - private def validateHeaderValue(value: String): Boolean = { - value.contains('\r') || value.contains('\n') || value.contains('\u0000') - } - private def nettyHeadersIterator(headers: HttpHeaders): Iterator[Header] = new AbstractIterator[Header] { private val nettyIterator = headers.iteratorCharSequence() @@ -103,10 +99,6 @@ private[netty] object Conversions { while (iter.hasNext) { val header = iter.next() val name = header.headerName - val value = header.renderedValueAsCharSequence.toString - if (validateHeaderValue(value)) { - throw new IllegalArgumentException(s"Invalid header value containing prohibited characters in header $name") - } if (name == setCookieName) { nettyHeaders.add(name, header.renderedValueAsCharSequence) } else { From b609cd99a5fac147915647881d6b7c8aacfe902b Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 23:10:34 +0530 Subject: [PATCH 11/12] fix --- .../test/scala/zio/http/endpoint/NotFoundSpec.scala | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala b/zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala index dc8860c653..b09ba78fb3 100644 --- a/zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala @@ -52,7 +52,7 @@ object NotFoundSpec extends ZIOHttpSpec { }, test("on wrong method") { check(Gen.int, Gen.int, Gen.alphaNumericString) { (userId, postId, name) => - val testRoutes = test404( + val testRoutes = test405( Routes( Endpoint(GET / "users" / int("userId")) .out[String] @@ -87,4 +87,15 @@ object NotFoundSpec extends ZIOHttpSpec { result = response.status == Status.NotFound } yield assertTrue(result) } + + def test405[R](service: Routes[R, Nothing])( + url: String, + method: Method, + ): ZIO[R, Response, TestResult] = { + val request = Request(method = method, url = URL.decode(url).toOption.get) + for { + response <- service.runZIO(request) + result = response.status == Status.MethodNotAllowed + } yield assertTrue(result) + } } From 62e78a36b88153a26a337eb8c7f610a189dc2f22 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Wed, 2 Oct 2024 08:40:36 +0530 Subject: [PATCH 12/12] feat(conformance): add review comments --- .../netty/server/ServerInboundHandler.scala | 21 +++++---- .../test/scala/zio/http/ConformanceSpec.scala | 2 +- .../src/main/scala/zio/http/Routes.scala | 46 ++++++++++--------- 3 files changed, 37 insertions(+), 32 deletions(-) diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala index d87261e25b..37a63707ae 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala @@ -116,16 +116,16 @@ private[zio] final case class ServerInboundHandler( } private def validateHostHeader(req: Request): Boolean = { - req.headers.get("Host") match { - case Some(host) => - val parts = host.split(":") - val hostname = parts(0) - val isValidHost = validateHostname(hostname) - val isValidPort = parts.length == 1 || (parts.length == 2 && parts(1).forall(_.isDigit)) - val isValid = isValidHost && isValidPort - isValid - case None => - false + val host = req.headers.get("Host").getOrElse(null) + if (host != null) { + val parts = host.split(":") + val hostname = parts(0) + val isValidHost = validateHostname(hostname) + val isValidPort = parts.length == 1 || (parts.length == 2 && parts(1).forall(_.isDigit)) + val isValid = isValidHost && isValidPort + isValid + } else { + false } } @@ -143,6 +143,7 @@ private[zio] final case class ServerInboundHandler( override def exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable): Unit = cause match { + case ioe: IOException if { val msg = ioe.getMessage (msg ne null) && msg.contains("Connection reset") diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala index fa80d17dd9..f2ae2abd30 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala @@ -1,7 +1,7 @@ package zio.http -import java.time.format.DateTimeFormatter import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter import zio._ import zio.test.Assertion._ diff --git a/zio-http/shared/src/main/scala/zio/http/Routes.scala b/zio-http/shared/src/main/scala/zio/http/Routes.scala index 642044ca0e..8695ebd2ea 100644 --- a/zio-http/shared/src/main/scala/zio/http/Routes.scala +++ b/zio-http/shared/src/main/scala/zio/http/Routes.scala @@ -249,32 +249,36 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s Handler .fromFunctionHandler[Request] { req => val chunk = tree.get(req.method, req.path) - val allowedMethods = tree.getAllMethods(req.path) + def allowedMethods = tree.getAllMethods(req.path) req.method match { case Method.CUSTOM(_) => Handler.notImplemented case _ => - chunk.length match { - case 0 => - if (allowedMethods.nonEmpty) { - val allowHeader = Header.Allow(NonEmptyChunk.fromIterableOption(allowedMethods).get) - Handler.methodNotAllowed.addHeader(allowHeader) - } else { - Handler.notFound - } - case 1 => chunk(0) - case n => // TODO: Support precomputed fallback among all chunk elements - var acc = chunk(0) - var i = 1 - while (i < n) { - val h = chunk(i) - acc = acc.catchAll { response => - if (response.status == Status.NotFound) h - else Handler.fail(response) + if (chunk.isEmpty) { + if (allowedMethods.isEmpty) { + // If no methods are allowed for the path, return 404 Not Found + Handler.notFound + } else { + // If there are allowed methods for the path but none match the request method, return 405 Method Not Allowed + val allowHeader = Header.Allow(NonEmptyChunk.fromIterableOption(allowedMethods).get) + Handler.methodNotAllowed.addHeader(allowHeader) + } + } else { + chunk.length match { + case 1 => chunk(0) + case n => // TODO: Support precomputed fallback among all chunk elements + var acc = chunk(0) + var i = 1 + while (i < n) { + val h = chunk(i) + acc = acc.catchAll { response => + if (response.status == Status.NotFound) h + else Handler.fail(response) + } + i += 1 } - i += 1 - } - acc + acc + } } } }