From 2812a6c0bcd5c85d43583ff5a66e417ed41b4f62 Mon Sep 17 00:00:00 2001 From: Giorgio Marinelli Date: Sun, 7 Apr 2024 19:10:51 +0200 Subject: [PATCH 01/11] Add the possibility to set the maximum number of headers --- http-client/Network/HTTP/Client.hs | 6 +++++ http-client/Network/HTTP/Client/Core.hs | 2 +- http-client/Network/HTTP/Client/Headers.hs | 17 ++++++++++--- http-client/Network/HTTP/Client/Manager.hs | 6 ++++- http-client/Network/HTTP/Client/Response.hs | 11 ++++---- http-client/Network/HTTP/Client/Types.hs | 25 +++++++++++++++++-- .../Network/HTTP/Client/HeadersSpec.hs | 12 ++++----- .../Network/HTTP/Client/ResponseSpec.hs | 2 +- http-conduit/Network/HTTP/Simple.hs | 1 - 9 files changed, 61 insertions(+), 21 deletions(-) diff --git a/http-client/Network/HTTP/Client.hs b/http-client/Network/HTTP/Client.hs index 97ba8c1a..d2eab8dc 100644 --- a/http-client/Network/HTTP/Client.hs +++ b/http-client/Network/HTTP/Client.hs @@ -113,6 +113,7 @@ module Network.HTTP.Client , managerSetInsecureProxy , managerSetSecureProxy , managerSetMaxHeaderLength + , managerSetMaxNumberHeaders , ProxyOverride , proxyFromRequest , noProxy @@ -326,6 +327,11 @@ managerSetMaxHeaderLength :: Int -> ManagerSettings -> ManagerSettings managerSetMaxHeaderLength l manager = manager { managerMaxHeaderLength = Just $ MaxHeaderLength l } +-- @since 0.7.18 +managerSetMaxNumberHeaders :: Int -> ManagerSettings -> ManagerSettings +managerSetMaxNumberHeaders l manager = manager + { managerMaxNumberHeaders = Just $ MaxNumberHeaders l } + -- $example1 -- = Example Usage -- diff --git a/http-client/Network/HTTP/Client/Core.hs b/http-client/Network/HTTP/Client/Core.hs index 777384f7..016904f5 100644 --- a/http-client/Network/HTTP/Client/Core.hs +++ b/http-client/Network/HTTP/Client/Core.hs @@ -105,7 +105,7 @@ httpRaw' req0 m = do ex <- try $ do cont <- requestBuilder (dropProxyAuthSecure req) (managedResource mconn) - getResponse (mMaxHeaderLength m) timeout' req mconn cont + getResponse (mMaxHeaderLength m) (mMaxNumberHeaders m) timeout' req mconn cont case ex of -- Connection was reused, and might have been closed. Try again diff --git a/http-client/Network/HTTP/Client/Headers.hs b/http-client/Network/HTTP/Client/Headers.hs index fefe808c..0c8a1903 100644 --- a/http-client/Network/HTTP/Client/Headers.hs +++ b/http-client/Network/HTTP/Client/Headers.hs @@ -28,8 +28,8 @@ charColon = 58 charPeriod = 46 -parseStatusHeaders :: Maybe MaxHeaderLength -> Connection -> Maybe Int -> ([Header] -> IO ()) -> Maybe (IO ()) -> IO StatusHeaders -parseStatusHeaders mhl conn timeout' onEarlyHintHeaders cont +parseStatusHeaders :: Maybe MaxHeaderLength -> Maybe MaxNumberHeaders -> Connection -> Maybe Int -> ([Header] -> IO ()) -> Maybe (IO ()) -> IO StatusHeaders +parseStatusHeaders mhl mnh conn timeout' onEarlyHintHeaders cont | Just k <- cont = getStatusExpectContinue k | otherwise = getStatus where @@ -91,9 +91,18 @@ parseStatusHeaders mhl conn timeout' onEarlyHintHeaders cont Just (i, "") -> Just i _ -> Nothing + guardMaxNumberHeaders :: Int -> IO () + guardMaxNumberHeaders count = case fmap unMaxNumberHeaders mnh of + -- We reached the maximum number of headers, let's throw an error + Just n | count >= n -> throwHttp OverlongHeaders + -- We didn't reach the maximum number of headers yet + Just _ -> pure () + -- We do not have any limit on the number of headers + Nothing -> pure () + parseHeaders :: Int -> ([Header] -> [Header]) -> IO [Header] - parseHeaders 100 _ = throwHttp OverlongHeaders parseHeaders count front = do + guardMaxNumberHeaders count line <- connectionReadLine mhl conn if S.null line then return $ front [] @@ -107,8 +116,8 @@ parseStatusHeaders mhl conn timeout' onEarlyHintHeaders cont parseHeaders count front parseEarlyHintHeadersUntilFailure :: Int -> ([Header] -> [Header]) -> IO [Header] - parseEarlyHintHeadersUntilFailure 100 _ = throwHttp OverlongHeaders parseEarlyHintHeadersUntilFailure count front = do + guardMaxNumberHeaders count line <- connectionReadLine mhl conn if S.null line then return $ front [] diff --git a/http-client/Network/HTTP/Client/Manager.hs b/http-client/Network/HTTP/Client/Manager.hs index fc21ac90..6d0de348 100644 --- a/http-client/Network/HTTP/Client/Manager.hs +++ b/http-client/Network/HTTP/Client/Manager.hs @@ -93,6 +93,7 @@ defaultManagerSettings = ManagerSettings , managerProxyInsecure = defaultProxy , managerProxySecure = defaultProxy , managerMaxHeaderLength = Just $ MaxHeaderLength 4096 + , managerMaxNumberHeaders = Just $ MaxNumberHeaders 100 } -- | Create a 'Manager'. The @Manager@ will be shut down automatically via @@ -133,6 +134,7 @@ newManager ms = do then httpsProxy req else httpProxy req , mMaxHeaderLength = managerMaxHeaderLength ms + , mMaxNumberHeaders = managerMaxNumberHeaders ms } return manager @@ -259,7 +261,9 @@ mkCreateConnection ms = do , "\r\n" ] parse conn = do - StatusHeaders status _ _ _ <- parseStatusHeaders (managerMaxHeaderLength ms) conn Nothing (\_ -> return ()) Nothing + let mhl = managerMaxHeaderLength ms + mnh = managerMaxNumberHeaders ms + StatusHeaders status _ _ _ <- parseStatusHeaders mhl mnh conn Nothing (\_ -> return ()) Nothing unless (status == status200) $ throwHttp $ ProxyConnectException ultHost ultPort status in tlsProxyConnection diff --git a/http-client/Network/HTTP/Client/Response.hs b/http-client/Network/HTTP/Client/Response.hs index ceb878ef..52c3eb58 100644 --- a/http-client/Network/HTTP/Client/Response.hs +++ b/http-client/Network/HTTP/Client/Response.hs @@ -81,10 +81,10 @@ getRedirectedRequest origReq req hs cookie_jar code mergeHeaders :: W.RequestHeaders -> W.RequestHeaders -> W.RequestHeaders mergeHeaders lhs rhs = nubBy (\(a, _) (a', _) -> a == a') (lhs ++ rhs) - + stripHeaders :: Request -> Request stripHeaders r = do - case (hostDiffer r, shouldStripOnlyIfHostDiffer) of + case (hostDiffer r, shouldStripOnlyIfHostDiffer) of (True, True) -> stripHeaders' r (True, False) -> stripHeaders' r (False, False) -> stripHeaders' r @@ -92,7 +92,7 @@ getRedirectedRequest origReq req hs cookie_jar code -- We need to check if we have omitted headers in previous -- request chain. Consider request chain: -- - -- 1. example-1.com + -- 1. example-1.com -- 2. example-2.com (we may have removed some headers here from 1) -- 3. example-1.com (since we are back at same host as 1, we need re-add stripped headers) -- @@ -114,14 +114,15 @@ lbsResponse res = do } getResponse :: Maybe MaxHeaderLength + -> Maybe MaxNumberHeaders -> Maybe Int -> Request -> Managed Connection -> Maybe (IO ()) -- ^ Action to run in case of a '100 Continue'. -> IO (Response BodyReader) -getResponse mhl timeout' req@(Request {..}) mconn cont = do +getResponse mhl mnh timeout' req@(Request {..}) mconn cont = do let conn = managedResource mconn - StatusHeaders s version earlyHs hs <- parseStatusHeaders mhl conn timeout' earlyHintHeadersReceived cont + StatusHeaders s version earlyHs hs <- parseStatusHeaders mhl mnh conn timeout' earlyHintHeadersReceived cont let mcl = lookup "content-length" hs >>= readPositiveInt . S8.unpack isChunked = ("transfer-encoding", CI.mk "chunked") `elem` map (second CI.mk) hs diff --git a/http-client/Network/HTTP/Client/Types.hs b/http-client/Network/HTTP/Client/Types.hs index 646dac79..1f9f4708 100644 --- a/http-client/Network/HTTP/Client/Types.hs +++ b/http-client/Network/HTTP/Client/Types.hs @@ -39,6 +39,7 @@ module Network.HTTP.Client.Types , ResponseTimeout (..) , ProxySecureMode (..) , MaxHeaderLength (..) + , MaxNumberHeaders (..) ) where import qualified Data.Typeable as T (Typeable) @@ -821,6 +822,17 @@ data ManagerSettings = ManagerSettings -- -- Since 0.4.7 , managerMaxHeaderLength :: Maybe MaxHeaderLength + -- ^ TODO + -- + -- Default: TODO + -- + -- @since TODO + , managerMaxNumberHeaders :: Maybe MaxNumberHeaders + -- ^ TODO + -- + -- Default: TODO + -- + -- @since 0.7.18 } deriving T.Typeable @@ -845,9 +857,10 @@ data Manager = Manager , mWrapException :: forall a. Request -> IO a -> IO a , mModifyRequest :: Request -> IO Request , mSetProxy :: Request -> Request - , mModifyResponse :: Response BodyReader -> IO (Response BodyReader) + , mModifyResponse :: Response BodyReader -> IO (Response BodyReader) -- ^ See 'managerProxy' , mMaxHeaderLength :: Maybe MaxHeaderLength + , mMaxNumberHeaders :: Maybe MaxNumberHeaders } deriving T.Typeable @@ -906,4 +919,12 @@ data StreamFileStatus = StreamFileStatus newtype MaxHeaderLength = MaxHeaderLength { unMaxHeaderLength :: Int } - deriving (Eq, Show) + deriving (Eq, Show, Ord, T.Typeable) + +-- | The maximum number of header lines. +-- +-- @since TODO +newtype MaxNumberHeaders = MaxNumberHeaders + { unMaxNumberHeaders :: Int + } + deriving (Eq, Show, Ord, T.Typeable) diff --git a/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs b/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs index 0d242d44..eb0b04c1 100644 --- a/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs +++ b/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs @@ -23,7 +23,7 @@ spec = describe "HeadersSpec" $ do , "\nignored" ] (connection, _, _) <- dummyConnection input - statusHeaders <- parseStatusHeaders Nothing connection Nothing (\_ -> return ()) Nothing + statusHeaders <- parseStatusHeaders Nothing Nothing connection Nothing (\_ -> return ()) Nothing statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) mempty [ ("foo", "bar") , ("baz", "bin") @@ -37,7 +37,7 @@ spec = describe "HeadersSpec" $ do ] (conn, out, _) <- dummyConnection input let sendBody = connectionWrite conn "data" - statusHeaders <- parseStatusHeaders Nothing conn Nothing (\_ -> return ()) (Just sendBody) + statusHeaders <- parseStatusHeaders Nothing Nothing conn Nothing (\_ -> return ()) (Just sendBody) statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [] [ ("foo", "bar") ] out >>= (`shouldBe` ["data"]) @@ -47,7 +47,7 @@ spec = describe "HeadersSpec" $ do ] (conn, out, _) <- dummyConnection input let sendBody = connectionWrite conn "data" - statusHeaders <- parseStatusHeaders Nothing conn Nothing (\_ -> return ()) (Just sendBody) + statusHeaders <- parseStatusHeaders Nothing Nothing conn Nothing (\_ -> return ()) (Just sendBody) statusHeaders `shouldBe` StatusHeaders status417 (HttpVersion 1 1) [] [] out >>= (`shouldBe` []) @@ -59,7 +59,7 @@ spec = describe "HeadersSpec" $ do , "result" ] (conn, out, inp) <- dummyConnection input - statusHeaders <- parseStatusHeaders Nothing conn Nothing (\_ -> return ()) Nothing + statusHeaders <- parseStatusHeaders Nothing Nothing conn Nothing (\_ -> return ()) Nothing statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [] [ ("foo", "bar") ] out >>= (`shouldBe` []) inp >>= (`shouldBe` ["result"]) @@ -78,7 +78,7 @@ spec = describe "HeadersSpec" $ do callbackResults :: MVar (Seq.Seq [Header]) <- newMVar mempty let onEarlyHintHeader h = modifyMVar_ callbackResults (return . (Seq.|> h)) - statusHeaders <- parseStatusHeaders Nothing conn Nothing onEarlyHintHeader Nothing + statusHeaders <- parseStatusHeaders Nothing Nothing conn Nothing onEarlyHintHeader Nothing statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [("Link", "") , ("Link", "") @@ -110,7 +110,7 @@ spec = describe "HeadersSpec" $ do callbackResults :: MVar (Seq.Seq [Header]) <- newMVar mempty let onEarlyHintHeader h = modifyMVar_ callbackResults (return . (Seq.|> h)) - statusHeaders <- parseStatusHeaders Nothing conn Nothing onEarlyHintHeader Nothing + statusHeaders <- parseStatusHeaders Nothing Nothing conn Nothing onEarlyHintHeader Nothing statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [("Link", "") , ("Link", "") diff --git a/http-client/test-nonet/Network/HTTP/Client/ResponseSpec.hs b/http-client/test-nonet/Network/HTTP/Client/ResponseSpec.hs index 0a5e3abe..cdb8d8ec 100644 --- a/http-client/test-nonet/Network/HTTP/Client/ResponseSpec.hs +++ b/http-client/test-nonet/Network/HTTP/Client/ResponseSpec.hs @@ -16,7 +16,7 @@ main = hspec spec spec :: Spec spec = describe "ResponseSpec" $ do - let getResponse' conn = getResponse Nothing Nothing req (dummyManaged conn) Nothing + let getResponse' conn = getResponse Nothing Nothing Nothing req (dummyManaged conn) Nothing req = parseRequest_ "http://localhost" it "basic" $ do (conn, _, _) <- dummyConnection diff --git a/http-conduit/Network/HTTP/Simple.hs b/http-conduit/Network/HTTP/Simple.hs index 25672f87..e46005d6 100644 --- a/http-conduit/Network/HTTP/Simple.hs +++ b/http-conduit/Network/HTTP/Simple.hs @@ -110,7 +110,6 @@ import qualified Data.Aeson as A import qualified Data.Traversable as T import Control.Exception (throw, throwIO, Exception) -import Data.Monoid import Data.Typeable (Typeable) import qualified Data.Conduit as C import Data.Conduit (runConduit, (.|), ConduitM) From 1411cf0fa4f285978f3edf6e3d9e873321cc6fd0 Mon Sep 17 00:00:00 2001 From: Giorgio Marinelli Date: Thu, 12 Dec 2024 19:12:22 +0100 Subject: [PATCH 02/11] Improve comments and update changelog --- http-client/ChangeLog.md | 4 ++++ http-client/Network/HTTP/Client/Types.hs | 16 +++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/http-client/ChangeLog.md b/http-client/ChangeLog.md index 0a99830a..5f1405b7 100644 --- a/http-client/ChangeLog.md +++ b/http-client/ChangeLog.md @@ -1,5 +1,9 @@ # Changelog for http-client +## 0.7.18 + +* Add the `managerSetMaxNumberHeaders` function to the `Client` module to configure `managerMaxNumberHeaders` in `ManagerSettings`. + ## 0.7.17 * Add `managerSetMaxHeaderLength` to `Client` to change `ManagerSettings` `MaxHeaderLength`. diff --git a/http-client/Network/HTTP/Client/Types.hs b/http-client/Network/HTTP/Client/Types.hs index 1f9f4708..7967a0aa 100644 --- a/http-client/Network/HTTP/Client/Types.hs +++ b/http-client/Network/HTTP/Client/Types.hs @@ -822,15 +822,17 @@ data ManagerSettings = ManagerSettings -- -- Since 0.4.7 , managerMaxHeaderLength :: Maybe MaxHeaderLength - -- ^ TODO + -- ^ Configure the maximum size, in bytes, of an HTTP header field. + -- Set it to `Nothing` to remove this limit (eg: for debugging purposes). -- - -- Default: TODO + -- Default: 4096 -- - -- @since TODO + -- @since 0.7.17 , managerMaxNumberHeaders :: Maybe MaxNumberHeaders - -- ^ TODO + -- ^ Configure the maximum number of HTTP header fields. + -- Set it to `Nothing` to remove this limit (eg: for debugging purposes). -- - -- Default: TODO + -- Default: 100 -- -- @since 0.7.18 } @@ -921,9 +923,9 @@ newtype MaxHeaderLength = MaxHeaderLength } deriving (Eq, Show, Ord, T.Typeable) --- | The maximum number of header lines. +-- | The maximum number of header fields. -- --- @since TODO +-- @since 0.7.18 newtype MaxNumberHeaders = MaxNumberHeaders { unMaxNumberHeaders :: Int } From 74d50486c01b0d4ee484d15856826ea949fbfc08 Mon Sep 17 00:00:00 2001 From: Giorgio Marinelli Date: Thu, 12 Dec 2024 19:23:27 +0100 Subject: [PATCH 03/11] Fix comments --- http-client/Network/HTTP/Client/Types.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http-client/Network/HTTP/Client/Types.hs b/http-client/Network/HTTP/Client/Types.hs index 7967a0aa..c148b7d1 100644 --- a/http-client/Network/HTTP/Client/Types.hs +++ b/http-client/Network/HTTP/Client/Types.hs @@ -823,14 +823,14 @@ data ManagerSettings = ManagerSettings -- Since 0.4.7 , managerMaxHeaderLength :: Maybe MaxHeaderLength -- ^ Configure the maximum size, in bytes, of an HTTP header field. - -- Set it to `Nothing` to remove this limit (eg: for debugging purposes). + -- Set it to 'Nothing' to remove this limit (eg: for debugging purposes). -- -- Default: 4096 -- -- @since 0.7.17 , managerMaxNumberHeaders :: Maybe MaxNumberHeaders -- ^ Configure the maximum number of HTTP header fields. - -- Set it to `Nothing` to remove this limit (eg: for debugging purposes). + -- Set it to 'Nothing' to remove this limit (eg: for debugging purposes). -- -- Default: 100 -- From 4916257518d29ec4faf23e4c58ee712b09196598 Mon Sep 17 00:00:00 2001 From: Giorgio Marinelli Date: Fri, 13 Dec 2024 09:20:09 +0100 Subject: [PATCH 04/11] Bump http-client version --- http-client/http-client.cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http-client/http-client.cabal b/http-client/http-client.cabal index 85862820..ee94b6c8 100644 --- a/http-client/http-client.cabal +++ b/http-client/http-client.cabal @@ -1,5 +1,5 @@ name: http-client -version: 0.7.17 +version: 0.7.18 synopsis: An HTTP client engine description: Hackage documentation generation is not reliable. For up to date documentation, please see: . homepage: https://github.com/snoyberg/http-client From c3312b42871d1209b234df90c396aef6ab9e1d31 Mon Sep 17 00:00:00 2001 From: Giorgio Marinelli Date: Fri, 13 Dec 2024 18:46:24 +0100 Subject: [PATCH 05/11] Use Word in MaxHeaderLength and MaxNumberHeaders --- http-client/Network/HTTP/Client.hs | 10 +++--- http-client/Network/HTTP/Client/Body.hs | 2 +- http-client/Network/HTTP/Client/Connection.hs | 15 +++++---- http-client/Network/HTTP/Client/Headers.hs | 23 ++++++-------- http-client/Network/HTTP/Client/Manager.hs | 4 +-- http-client/Network/HTTP/Client/Response.hs | 4 +-- http-client/Network/HTTP/Client/Types.hs | 31 ++++++++++++------- .../Network/HTTP/Client/BodySpec.hs | 16 +++++----- .../Network/HTTP/Client/HeadersSpec.hs | 12 +++---- .../Network/HTTP/Client/ResponseSpec.hs | 2 +- 10 files changed, 62 insertions(+), 57 deletions(-) diff --git a/http-client/Network/HTTP/Client.hs b/http-client/Network/HTTP/Client.hs index d2eab8dc..4182ee58 100644 --- a/http-client/Network/HTTP/Client.hs +++ b/http-client/Network/HTTP/Client.hs @@ -323,14 +323,14 @@ managerSetProxy :: ProxyOverride -> ManagerSettings -> ManagerSettings managerSetProxy po = managerSetInsecureProxy po . managerSetSecureProxy po -- @since 0.7.17 -managerSetMaxHeaderLength :: Int -> ManagerSettings -> ManagerSettings +managerSetMaxHeaderLength :: Word -> ManagerSettings -> ManagerSettings managerSetMaxHeaderLength l manager = manager - { managerMaxHeaderLength = Just $ MaxHeaderLength l } + { managerMaxHeaderLength = MaxHeaderLength l } -- @since 0.7.18 -managerSetMaxNumberHeaders :: Int -> ManagerSettings -> ManagerSettings -managerSetMaxNumberHeaders l manager = manager - { managerMaxNumberHeaders = Just $ MaxNumberHeaders l } +managerSetMaxNumberHeaders :: Word -> ManagerSettings -> ManagerSettings +managerSetMaxNumberHeaders n manager = manager + { managerMaxNumberHeaders = MaxNumberHeaders n } -- $example1 -- = Example Usage diff --git a/http-client/Network/HTTP/Client/Body.hs b/http-client/Network/HTTP/Client/Body.hs index a44834cb..09c6bc40 100644 --- a/http-client/Network/HTTP/Client/Body.hs +++ b/http-client/Network/HTTP/Client/Body.hs @@ -148,7 +148,7 @@ makeLengthReader cleanup count0 Connection {..} = do return bs makeChunkedReader - :: Maybe MaxHeaderLength + :: MaxHeaderLength -> IO () -- ^ cleanup -> Bool -- ^ raw -> Connection diff --git a/http-client/Network/HTTP/Client/Connection.hs b/http-client/Network/HTTP/Client/Connection.hs index a57da553..99a3502e 100644 --- a/http-client/Network/HTTP/Client/Connection.hs +++ b/http-client/Network/HTTP/Client/Connection.hs @@ -31,32 +31,31 @@ import Data.Function (fix) import Data.Maybe (listToMaybe) import Data.Word (Word8) -connectionReadLine :: Maybe MaxHeaderLength -> Connection -> IO ByteString +connectionReadLine :: MaxHeaderLength -> Connection -> IO ByteString connectionReadLine mhl conn = do bs <- connectionRead conn when (S.null bs) $ throwHttp IncompleteHeaders connectionReadLineWith mhl conn bs -- | Keep dropping input until a blank line is found. -connectionDropTillBlankLine :: Maybe MaxHeaderLength -> Connection -> IO () +connectionDropTillBlankLine :: MaxHeaderLength -> Connection -> IO () connectionDropTillBlankLine mhl conn = fix $ \loop -> do bs <- connectionReadLine mhl conn unless (S.null bs) loop -connectionReadLineWith :: Maybe MaxHeaderLength -> Connection -> ByteString -> IO ByteString +connectionReadLineWith :: MaxHeaderLength -> Connection -> ByteString -> IO ByteString connectionReadLineWith mhl conn bs0 = go bs0 id 0 where go bs front total = case S.break (== charLF) bs of (_, "") -> do - let total' = total + S.length bs - case fmap unMaxHeaderLength mhl of - Nothing -> pure () - Just n -> when (total' > n) $ throwHttp OverlongHeaders + when (total >= unMaxHeaderLength mhl && total /= 0) $ do + -- We reached the maximum length for an header field. + throwHttp OverlongHeaders bs' <- connectionRead conn when (S.null bs') $ throwHttp IncompleteHeaders - go bs' (front . (bs:)) total' + go bs' (front . (bs:)) (total + fromIntegral (S.length bs)) (x, S.drop 1 -> y) -> do unless (S.null y) $! connectionUnread conn y return $! killCR $! S.concat $! front [x] diff --git a/http-client/Network/HTTP/Client/Headers.hs b/http-client/Network/HTTP/Client/Headers.hs index 0c8a1903..82badf87 100644 --- a/http-client/Network/HTTP/Client/Headers.hs +++ b/http-client/Network/HTTP/Client/Headers.hs @@ -28,7 +28,7 @@ charColon = 58 charPeriod = 46 -parseStatusHeaders :: Maybe MaxHeaderLength -> Maybe MaxNumberHeaders -> Connection -> Maybe Int -> ([Header] -> IO ()) -> Maybe (IO ()) -> IO StatusHeaders +parseStatusHeaders :: MaxHeaderLength -> MaxNumberHeaders -> Connection -> Maybe Int -> ([Header] -> IO ()) -> Maybe (IO ()) -> IO StatusHeaders parseStatusHeaders mhl mnh conn timeout' onEarlyHintHeaders cont | Just k <- cont = getStatusExpectContinue k | otherwise = getStatus @@ -60,7 +60,7 @@ parseStatusHeaders mhl mnh conn timeout' onEarlyHintHeaders cont return $ Just $ StatusHeaders s' v' (earlyHeaders <> earlyHeaders') reqHeaders | otherwise -> (Just <$>) $ StatusHeaders s v mempty A.<$> parseHeaders 0 id - nextStatusLine :: Maybe MaxHeaderLength -> IO (Status, HttpVersion) + nextStatusLine :: MaxHeaderLength -> IO (Status, HttpVersion) nextStatusLine mhl = do -- Ensure that there is some data coming in. If not, we want to signal -- this as a connection problem and not a protocol problem. @@ -68,7 +68,7 @@ parseStatusHeaders mhl mnh conn timeout' onEarlyHintHeaders cont when (S.null bs) $ throwHttp NoResponseDataReceived connectionReadLineWith mhl conn bs >>= parseStatus mhl 3 - parseStatus :: Maybe MaxHeaderLength -> Int -> S.ByteString -> IO (Status, HttpVersion) + parseStatus :: MaxHeaderLength -> Int -> S.ByteString -> IO (Status, HttpVersion) parseStatus mhl i bs | S.null bs && i > 0 = connectionReadLine mhl conn >>= parseStatus mhl (i - 1) parseStatus _ _ bs = do let (ver, bs2) = S.break (== charSpace) bs @@ -91,16 +91,13 @@ parseStatusHeaders mhl mnh conn timeout' onEarlyHintHeaders cont Just (i, "") -> Just i _ -> Nothing - guardMaxNumberHeaders :: Int -> IO () - guardMaxNumberHeaders count = case fmap unMaxNumberHeaders mnh of - -- We reached the maximum number of headers, let's throw an error - Just n | count >= n -> throwHttp OverlongHeaders - -- We didn't reach the maximum number of headers yet - Just _ -> pure () - -- We do not have any limit on the number of headers - Nothing -> pure () + guardMaxNumberHeaders :: Word -> IO () + guardMaxNumberHeaders count = + when (count >= unMaxNumberHeaders mnh && count /= 0) $ do + -- We reached the maximum number of header fields. + throwHttp OverlongHeaders - parseHeaders :: Int -> ([Header] -> [Header]) -> IO [Header] + parseHeaders :: Word -> ([Header] -> [Header]) -> IO [Header] parseHeaders count front = do guardMaxNumberHeaders count line <- connectionReadLine mhl conn @@ -115,7 +112,7 @@ parseStatusHeaders mhl mnh conn timeout' onEarlyHintHeaders cont -- an exception, ignore it for robustness. parseHeaders count front - parseEarlyHintHeadersUntilFailure :: Int -> ([Header] -> [Header]) -> IO [Header] + parseEarlyHintHeadersUntilFailure :: Word -> ([Header] -> [Header]) -> IO [Header] parseEarlyHintHeadersUntilFailure count front = do guardMaxNumberHeaders count line <- connectionReadLine mhl conn diff --git a/http-client/Network/HTTP/Client/Manager.hs b/http-client/Network/HTTP/Client/Manager.hs index 6d0de348..abe92ff6 100644 --- a/http-client/Network/HTTP/Client/Manager.hs +++ b/http-client/Network/HTTP/Client/Manager.hs @@ -92,8 +92,8 @@ defaultManagerSettings = ManagerSettings , managerModifyResponse = return , managerProxyInsecure = defaultProxy , managerProxySecure = defaultProxy - , managerMaxHeaderLength = Just $ MaxHeaderLength 4096 - , managerMaxNumberHeaders = Just $ MaxNumberHeaders 100 + , managerMaxHeaderLength = 4096 + , managerMaxNumberHeaders = 100 } -- | Create a 'Manager'. The @Manager@ will be shut down automatically via diff --git a/http-client/Network/HTTP/Client/Response.hs b/http-client/Network/HTTP/Client/Response.hs index 52c3eb58..87de0298 100644 --- a/http-client/Network/HTTP/Client/Response.hs +++ b/http-client/Network/HTTP/Client/Response.hs @@ -113,8 +113,8 @@ lbsResponse res = do { responseBody = L.fromChunks bss } -getResponse :: Maybe MaxHeaderLength - -> Maybe MaxNumberHeaders +getResponse :: MaxHeaderLength + -> MaxNumberHeaders -> Maybe Int -> Request -> Managed Connection diff --git a/http-client/Network/HTTP/Client/Types.hs b/http-client/Network/HTTP/Client/Types.hs index c148b7d1..0ee47c08 100644 --- a/http-client/Network/HTTP/Client/Types.hs +++ b/http-client/Network/HTTP/Client/Types.hs @@ -1,8 +1,9 @@ {-# LANGUAGE CPP #-} {-# LANGUAGE DeriveDataTypeable #-} {-# LANGUAGE DeriveTraversable #-} -{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE GeneralizedNewtypeDeriving #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RankNTypes #-} module Network.HTTP.Client.Types ( BodyReader , Connection (..) @@ -39,7 +40,9 @@ module Network.HTTP.Client.Types , ResponseTimeout (..) , ProxySecureMode (..) , MaxHeaderLength (..) + , noMaxHeaderLength , MaxNumberHeaders (..) + , noMaxNumberHeaders ) where import qualified Data.Typeable as T (Typeable) @@ -821,16 +824,16 @@ data ManagerSettings = ManagerSettings -- Default: respect the @proxy@ value on the @Request@ itself. -- -- Since 0.4.7 - , managerMaxHeaderLength :: Maybe MaxHeaderLength + , managerMaxHeaderLength :: MaxHeaderLength -- ^ Configure the maximum size, in bytes, of an HTTP header field. - -- Set it to 'Nothing' to remove this limit (eg: for debugging purposes). + -- Set it to 0 to remove this limit (eg: for debugging purposes). -- -- Default: 4096 -- -- @since 0.7.17 - , managerMaxNumberHeaders :: Maybe MaxNumberHeaders + , managerMaxNumberHeaders :: MaxNumberHeaders -- ^ Configure the maximum number of HTTP header fields. - -- Set it to 'Nothing' to remove this limit (eg: for debugging purposes). + -- Set it to 0 to remove this limit (eg: for debugging purposes). -- -- Default: 100 -- @@ -861,8 +864,8 @@ data Manager = Manager , mSetProxy :: Request -> Request , mModifyResponse :: Response BodyReader -> IO (Response BodyReader) -- ^ See 'managerProxy' - , mMaxHeaderLength :: Maybe MaxHeaderLength - , mMaxNumberHeaders :: Maybe MaxNumberHeaders + , mMaxHeaderLength :: MaxHeaderLength + , mMaxNumberHeaders :: MaxNumberHeaders } deriving T.Typeable @@ -919,14 +922,20 @@ data StreamFileStatus = StreamFileStatus -- -- @since 0.7.14 newtype MaxHeaderLength = MaxHeaderLength - { unMaxHeaderLength :: Int + { unMaxHeaderLength :: Word } - deriving (Eq, Show, Ord, T.Typeable) + deriving (Eq, Show, Ord, Num, T.Typeable) + +noMaxHeaderLength :: MaxHeaderLength +noMaxHeaderLength = 0 -- | The maximum number of header fields. -- -- @since 0.7.18 newtype MaxNumberHeaders = MaxNumberHeaders - { unMaxNumberHeaders :: Int + { unMaxNumberHeaders :: Word } - deriving (Eq, Show, Ord, T.Typeable) + deriving (Eq, Show, Ord, Num, T.Typeable) + +noMaxNumberHeaders :: MaxNumberHeaders +noMaxNumberHeaders = 0 diff --git a/http-client/test-nonet/Network/HTTP/Client/BodySpec.hs b/http-client/test-nonet/Network/HTTP/Client/BodySpec.hs index a2f2e613..5bd28293 100644 --- a/http-client/test-nonet/Network/HTTP/Client/BodySpec.hs +++ b/http-client/test-nonet/Network/HTTP/Client/BodySpec.hs @@ -22,7 +22,7 @@ spec = describe "BodySpec" $ do (conn, _, input) <- dummyConnection [ "5\r\nhello\r\n6\r\n world\r\n0\r\n\r\nnot consumed" ] - reader <- makeChunkedReader Nothing (return ()) False conn + reader <- makeChunkedReader noMaxHeaderLength (return ()) False conn body <- brConsume reader S.concat body `shouldBe` "hello world" input' <- input @@ -33,7 +33,7 @@ spec = describe "BodySpec" $ do (conn, _, input) <- dummyConnection [ "5\r\nhello\r\n6\r\n world\r\n0\r\ntrailers-are: ignored\r\nbut: consumed\r\n\r\nnot consumed" ] - reader <- makeChunkedReader Nothing (return ()) False conn + reader <- makeChunkedReader noMaxHeaderLength (return ()) False conn body <- brConsume reader S.concat body `shouldBe` "hello world" input' <- input @@ -43,7 +43,7 @@ spec = describe "BodySpec" $ do it "chunked, pieces" $ do (conn, _, input) <- dummyConnection $ map S.singleton $ S.unpack "5\r\nhello\r\n6\r\n world\r\n0\r\n\r\nnot consumed" - reader <- makeChunkedReader Nothing (return ()) False conn + reader <- makeChunkedReader noMaxHeaderLength (return ()) False conn body <- brConsume reader S.concat body `shouldBe` "hello world" input' <- input @@ -53,7 +53,7 @@ spec = describe "BodySpec" $ do it "chunked, pieces, with trailers" $ do (conn, _, input) <- dummyConnection $ map S.singleton $ S.unpack "5\r\nhello\r\n6\r\n world\r\n0\r\ntrailers-are: ignored\r\nbut: consumed\r\n\r\nnot consumed" - reader <- makeChunkedReader Nothing (return ()) False conn + reader <- makeChunkedReader noMaxHeaderLength (return ()) False conn body <- brConsume reader S.concat body `shouldBe` "hello world" input' <- input @@ -64,7 +64,7 @@ spec = describe "BodySpec" $ do (conn, _, input) <- dummyConnection [ "5\r\nhello\r\n6\r\n world\r\n0\r\n\r\nnot consumed" ] - reader <- makeChunkedReader Nothing (return ()) True conn + reader <- makeChunkedReader noMaxHeaderLength (return ()) True conn body <- brConsume reader S.concat body `shouldBe` "5\r\nhello\r\n6\r\n world\r\n0\r\n\r\n" input' <- input @@ -75,7 +75,7 @@ spec = describe "BodySpec" $ do (conn, _, input) <- dummyConnection [ "5\r\nhello\r\n6\r\n world\r\n0\r\ntrailers-are: returned\r\nin-raw: body\r\n\r\nnot consumed" ] - reader <- makeChunkedReader Nothing (return ()) True conn + reader <- makeChunkedReader noMaxHeaderLength (return ()) True conn body <- brConsume reader S.concat body `shouldBe` "5\r\nhello\r\n6\r\n world\r\n0\r\ntrailers-are: returned\r\nin-raw: body\r\n\r\n" input' <- input @@ -85,7 +85,7 @@ spec = describe "BodySpec" $ do it "chunked, pieces, raw" $ do (conn, _, input) <- dummyConnection $ map S.singleton $ S.unpack "5\r\nhello\r\n6\r\n world\r\n0\r\n\r\nnot consumed" - reader <- makeChunkedReader Nothing (return ()) True conn + reader <- makeChunkedReader noMaxHeaderLength (return ()) True conn body <- brConsume reader S.concat body `shouldBe` "5\r\nhello\r\n6\r\n world\r\n0\r\n\r\n" input' <- input @@ -95,7 +95,7 @@ spec = describe "BodySpec" $ do it "chunked, pieces, raw, with trailers" $ do (conn, _, input) <- dummyConnection $ map S.singleton $ S.unpack "5\r\nhello\r\n6\r\n world\r\n0\r\ntrailers-are: returned\r\nin-raw: body\r\n\r\nnot consumed" - reader <- makeChunkedReader Nothing (return ()) True conn + reader <- makeChunkedReader noMaxHeaderLength (return ()) True conn body <- brConsume reader S.concat body `shouldBe` "5\r\nhello\r\n6\r\n world\r\n0\r\ntrailers-are: returned\r\nin-raw: body\r\n\r\n" input' <- input diff --git a/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs b/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs index eb0b04c1..955505d3 100644 --- a/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs +++ b/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs @@ -23,7 +23,7 @@ spec = describe "HeadersSpec" $ do , "\nignored" ] (connection, _, _) <- dummyConnection input - statusHeaders <- parseStatusHeaders Nothing Nothing connection Nothing (\_ -> return ()) Nothing + statusHeaders <- parseStatusHeaders noMaxHeaderLength noMaxNumberHeaders connection Nothing (\_ -> return ()) Nothing statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) mempty [ ("foo", "bar") , ("baz", "bin") @@ -37,7 +37,7 @@ spec = describe "HeadersSpec" $ do ] (conn, out, _) <- dummyConnection input let sendBody = connectionWrite conn "data" - statusHeaders <- parseStatusHeaders Nothing Nothing conn Nothing (\_ -> return ()) (Just sendBody) + statusHeaders <- parseStatusHeaders noMaxHeaderLength noMaxNumberHeaders conn Nothing (\_ -> return ()) (Just sendBody) statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [] [ ("foo", "bar") ] out >>= (`shouldBe` ["data"]) @@ -47,7 +47,7 @@ spec = describe "HeadersSpec" $ do ] (conn, out, _) <- dummyConnection input let sendBody = connectionWrite conn "data" - statusHeaders <- parseStatusHeaders Nothing Nothing conn Nothing (\_ -> return ()) (Just sendBody) + statusHeaders <- parseStatusHeaders noMaxHeaderLength noMaxNumberHeaders conn Nothing (\_ -> return ()) (Just sendBody) statusHeaders `shouldBe` StatusHeaders status417 (HttpVersion 1 1) [] [] out >>= (`shouldBe` []) @@ -59,7 +59,7 @@ spec = describe "HeadersSpec" $ do , "result" ] (conn, out, inp) <- dummyConnection input - statusHeaders <- parseStatusHeaders Nothing Nothing conn Nothing (\_ -> return ()) Nothing + statusHeaders <- parseStatusHeaders noMaxHeaderLength noMaxNumberHeaders conn Nothing (\_ -> return ()) Nothing statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [] [ ("foo", "bar") ] out >>= (`shouldBe` []) inp >>= (`shouldBe` ["result"]) @@ -78,7 +78,7 @@ spec = describe "HeadersSpec" $ do callbackResults :: MVar (Seq.Seq [Header]) <- newMVar mempty let onEarlyHintHeader h = modifyMVar_ callbackResults (return . (Seq.|> h)) - statusHeaders <- parseStatusHeaders Nothing Nothing conn Nothing onEarlyHintHeader Nothing + statusHeaders <- parseStatusHeaders noMaxHeaderLength noMaxNumberHeaders conn Nothing onEarlyHintHeader Nothing statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [("Link", "") , ("Link", "") @@ -110,7 +110,7 @@ spec = describe "HeadersSpec" $ do callbackResults :: MVar (Seq.Seq [Header]) <- newMVar mempty let onEarlyHintHeader h = modifyMVar_ callbackResults (return . (Seq.|> h)) - statusHeaders <- parseStatusHeaders Nothing Nothing conn Nothing onEarlyHintHeader Nothing + statusHeaders <- parseStatusHeaders noMaxHeaderLength noMaxNumberHeaders conn Nothing onEarlyHintHeader Nothing statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [("Link", "") , ("Link", "") diff --git a/http-client/test-nonet/Network/HTTP/Client/ResponseSpec.hs b/http-client/test-nonet/Network/HTTP/Client/ResponseSpec.hs index cdb8d8ec..3728fd4f 100644 --- a/http-client/test-nonet/Network/HTTP/Client/ResponseSpec.hs +++ b/http-client/test-nonet/Network/HTTP/Client/ResponseSpec.hs @@ -16,7 +16,7 @@ main = hspec spec spec :: Spec spec = describe "ResponseSpec" $ do - let getResponse' conn = getResponse Nothing Nothing Nothing req (dummyManaged conn) Nothing + let getResponse' conn = getResponse noMaxHeaderLength noMaxNumberHeaders Nothing req (dummyManaged conn) Nothing req = parseRequest_ "http://localhost" it "basic" $ do (conn, _, _) <- dummyConnection From d6644ace032c46b64a730e01b85ffab0d6ca0c03 Mon Sep 17 00:00:00 2001 From: Giorgio Marinelli Date: Fri, 13 Dec 2024 18:51:04 +0100 Subject: [PATCH 06/11] Update http-client changelog --- http-client/ChangeLog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/http-client/ChangeLog.md b/http-client/ChangeLog.md index 5f1405b7..3fad9bac 100644 --- a/http-client/ChangeLog.md +++ b/http-client/ChangeLog.md @@ -3,6 +3,7 @@ ## 0.7.18 * Add the `managerSetMaxNumberHeaders` function to the `Client` module to configure `managerMaxNumberHeaders` in `ManagerSettings`. +* Use `Word` in the `MaxHeaderLength` and `MaxNumberHeaders` types. ## 0.7.17 From b2e496a2cc445200f108eea006e954c64147a476 Mon Sep 17 00:00:00 2001 From: Giorgio Marinelli Date: Sun, 15 Dec 2024 22:54:17 +0100 Subject: [PATCH 07/11] Revert some changes; add TooManyHeaders exception; add tests --- http-client/Network/HTTP/Client/Connection.hs | 5 ++-- http-client/Network/HTTP/Client/Headers.hs | 4 +-- http-client/Network/HTTP/Client/Types.hs | 20 ++++++------- http-conduit/test/main.hs | 30 ++++++++++++++++++- 4 files changed, 44 insertions(+), 15 deletions(-) diff --git a/http-client/Network/HTTP/Client/Connection.hs b/http-client/Network/HTTP/Client/Connection.hs index 99a3502e..673b0d61 100644 --- a/http-client/Network/HTTP/Client/Connection.hs +++ b/http-client/Network/HTTP/Client/Connection.hs @@ -50,12 +50,13 @@ connectionReadLineWith mhl conn bs0 = go bs front total = case S.break (== charLF) bs of (_, "") -> do - when (total >= unMaxHeaderLength mhl && total /= 0) $ do + let total' = total + fromIntegral (S.length bs) + when (total > unMaxHeaderLength mhl) $ do -- We reached the maximum length for an header field. throwHttp OverlongHeaders bs' <- connectionRead conn when (S.null bs') $ throwHttp IncompleteHeaders - go bs' (front . (bs:)) (total + fromIntegral (S.length bs)) + go bs' (front . (bs:)) total' (x, S.drop 1 -> y) -> do unless (S.null y) $! connectionUnread conn y return $! killCR $! S.concat $! front [x] diff --git a/http-client/Network/HTTP/Client/Headers.hs b/http-client/Network/HTTP/Client/Headers.hs index 82badf87..7b585838 100644 --- a/http-client/Network/HTTP/Client/Headers.hs +++ b/http-client/Network/HTTP/Client/Headers.hs @@ -93,9 +93,9 @@ parseStatusHeaders mhl mnh conn timeout' onEarlyHintHeaders cont guardMaxNumberHeaders :: Word -> IO () guardMaxNumberHeaders count = - when (count >= unMaxNumberHeaders mnh && count /= 0) $ do + when (count >= unMaxNumberHeaders mnh) $ do -- We reached the maximum number of header fields. - throwHttp OverlongHeaders + throwHttp TooManyHeaders parseHeaders :: Word -> ([Header] -> [Header]) -> IO [Header] parseHeaders count front = do diff --git a/http-client/Network/HTTP/Client/Types.hs b/http-client/Network/HTTP/Client/Types.hs index 0ee47c08..8473bf9e 100644 --- a/http-client/Network/HTTP/Client/Types.hs +++ b/http-client/Network/HTTP/Client/Types.hs @@ -151,12 +151,14 @@ data HttpExceptionContent -- -- @since 0.5.0 | OverlongHeaders - -- ^ Either too many headers, or too many total bytes in a - -- single header, were returned by the server, and the - -- memory exhaustion protection in this library has kicked - -- in. + -- ^ Too many total bytes in a single header field were + -- returned by the server. -- -- @since 0.5.0 + | TooManyHeaders + -- ^ Too many header fields were returned by the server, + -- + -- @since 0.7.18 | ResponseTimeout -- ^ The server took too long to return a response. This can -- be altered via 'responseTimeout' or @@ -826,14 +828,12 @@ data ManagerSettings = ManagerSettings -- Since 0.4.7 , managerMaxHeaderLength :: MaxHeaderLength -- ^ Configure the maximum size, in bytes, of an HTTP header field. - -- Set it to 0 to remove this limit (eg: for debugging purposes). -- -- Default: 4096 -- -- @since 0.7.17 , managerMaxNumberHeaders :: MaxNumberHeaders -- ^ Configure the maximum number of HTTP header fields. - -- Set it to 0 to remove this limit (eg: for debugging purposes). -- -- Default: 100 -- @@ -924,10 +924,10 @@ data StreamFileStatus = StreamFileStatus newtype MaxHeaderLength = MaxHeaderLength { unMaxHeaderLength :: Word } - deriving (Eq, Show, Ord, Num, T.Typeable) + deriving (Eq, Show, Ord, Num, Enum, Bounded, T.Typeable) noMaxHeaderLength :: MaxHeaderLength -noMaxHeaderLength = 0 +noMaxHeaderLength = maxBound -- | The maximum number of header fields. -- @@ -935,7 +935,7 @@ noMaxHeaderLength = 0 newtype MaxNumberHeaders = MaxNumberHeaders { unMaxNumberHeaders :: Word } - deriving (Eq, Show, Ord, Num, T.Typeable) + deriving (Eq, Show, Ord, Num, Enum, Bounded, T.Typeable) noMaxNumberHeaders :: MaxNumberHeaders -noMaxNumberHeaders = 0 +noMaxNumberHeaders = maxBound diff --git a/http-conduit/test/main.hs b/http-conduit/test/main.hs index 632f5f81..43144ee2 100644 --- a/http-conduit/test/main.hs +++ b/http-conduit/test/main.hs @@ -8,7 +8,8 @@ import qualified Data.ByteString.Lazy.Char8 as L8 import Test.HUnit import Network.Wai hiding (requestBody) import Network.Wai.Conduit (responseSource, sourceRequestBody) -import Network.HTTP.Client (streamFile) +import Network.HTTP.Client (streamFile, defaultManagerSettings) +import Network.HTTP.Client.Internal (managerMaxNumberHeaders) import System.IO.Temp (withSystemTempFile) import qualified Network.Wai as Wai import Network.Wai.Handler.Warp (runSettings, defaultSettings, setPort, setBeforeMainLoop, Settings, setTimeout) @@ -261,6 +262,18 @@ main = do let Just req1 = parseUrlThrow $ "http://127.0.0.1:" ++ show port _ <- httpLbs req1 manager return () + it "too many header fields" $ tooManyHeaderFields $ \port -> do + manager <- newManager tlsManagerSettings + let Just req1 = parseUrlThrow $ "http://127.0.0.1:" ++ show port + res1 <- try $ runResourceT $ http req1 manager + case res1 of + Left e -> show (e :: SomeException) @?= show (HttpExceptionRequest req1 TooManyHeaders) + _ -> error "Shouldn't have worked" + it "not too many header fields" $ notTooManyHeaderFields $ \port -> do + manager <- newManager tlsManagerSettings + let Just req1 = parseUrlThrow $ "http://127.0.0.1:" ++ show port + _ <- httpLbs req1 manager + return () describe "redirects" $ do it "doesn't double escape" $ redir $ \port -> do manager <- newManager tlsManagerSettings @@ -525,6 +538,21 @@ notOverLongHeaders = withCApp $ \app' -> do where src = sourceList $ [S.concat $ "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 16384\r\n\r\n" : ( take 16384 $ repeat "x")] +tooManyHeaderFields :: (Int -> IO ()) -> IO () +tooManyHeaderFields = + withCApp $ \app' -> runConduit $ src .| appSink app' + where + limit = fromEnum (managerMaxNumberHeaders defaultManagerSettings) + src = sourceList $ "HTTP/1.0 200 OK\r\n" : replicate limit "foo: bar\r\n" + +notTooManyHeaderFields :: (Int -> IO ()) -> IO () +notTooManyHeaderFields = withCApp $ \app' -> do + runConduit $ appSource app' .| CL.drop 1 + runConduit $ src .| appSink app' + where + limit = fromEnum (managerMaxNumberHeaders defaultManagerSettings) - 1 + src = sourceList $ ["HTTP/1.0 200 OK\r\n"] <> replicate limit "foo: bar\r\n" <> ["\r\n"] + redir :: (Int -> IO ()) -> IO () redir = withApp' redirApp From a104b38c2bdc59c75801b09fcfaa75b44c36565c Mon Sep 17 00:00:00 2001 From: Giorgio Marinelli Date: Mon, 16 Dec 2024 07:20:35 +0100 Subject: [PATCH 08/11] Switch back to Int --- http-client/Network/HTTP/Client.hs | 4 ++-- http-client/Network/HTTP/Client/Connection.hs | 2 +- http-client/Network/HTTP/Client/Headers.hs | 6 +++--- http-client/Network/HTTP/Client/Types.hs | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/http-client/Network/HTTP/Client.hs b/http-client/Network/HTTP/Client.hs index 4182ee58..d5cd9fe9 100644 --- a/http-client/Network/HTTP/Client.hs +++ b/http-client/Network/HTTP/Client.hs @@ -323,12 +323,12 @@ managerSetProxy :: ProxyOverride -> ManagerSettings -> ManagerSettings managerSetProxy po = managerSetInsecureProxy po . managerSetSecureProxy po -- @since 0.7.17 -managerSetMaxHeaderLength :: Word -> ManagerSettings -> ManagerSettings +managerSetMaxHeaderLength :: Int -> ManagerSettings -> ManagerSettings managerSetMaxHeaderLength l manager = manager { managerMaxHeaderLength = MaxHeaderLength l } -- @since 0.7.18 -managerSetMaxNumberHeaders :: Word -> ManagerSettings -> ManagerSettings +managerSetMaxNumberHeaders :: Int -> ManagerSettings -> ManagerSettings managerSetMaxNumberHeaders n manager = manager { managerMaxNumberHeaders = MaxNumberHeaders n } diff --git a/http-client/Network/HTTP/Client/Connection.hs b/http-client/Network/HTTP/Client/Connection.hs index 673b0d61..26db9638 100644 --- a/http-client/Network/HTTP/Client/Connection.hs +++ b/http-client/Network/HTTP/Client/Connection.hs @@ -50,7 +50,7 @@ connectionReadLineWith mhl conn bs0 = go bs front total = case S.break (== charLF) bs of (_, "") -> do - let total' = total + fromIntegral (S.length bs) + let total' = total + S.length bs when (total > unMaxHeaderLength mhl) $ do -- We reached the maximum length for an header field. throwHttp OverlongHeaders diff --git a/http-client/Network/HTTP/Client/Headers.hs b/http-client/Network/HTTP/Client/Headers.hs index 7b585838..d1ba2bfe 100644 --- a/http-client/Network/HTTP/Client/Headers.hs +++ b/http-client/Network/HTTP/Client/Headers.hs @@ -91,13 +91,13 @@ parseStatusHeaders mhl mnh conn timeout' onEarlyHintHeaders cont Just (i, "") -> Just i _ -> Nothing - guardMaxNumberHeaders :: Word -> IO () + guardMaxNumberHeaders :: Int -> IO () guardMaxNumberHeaders count = when (count >= unMaxNumberHeaders mnh) $ do -- We reached the maximum number of header fields. throwHttp TooManyHeaders - parseHeaders :: Word -> ([Header] -> [Header]) -> IO [Header] + parseHeaders :: Int -> ([Header] -> [Header]) -> IO [Header] parseHeaders count front = do guardMaxNumberHeaders count line <- connectionReadLine mhl conn @@ -112,7 +112,7 @@ parseStatusHeaders mhl mnh conn timeout' onEarlyHintHeaders cont -- an exception, ignore it for robustness. parseHeaders count front - parseEarlyHintHeadersUntilFailure :: Word -> ([Header] -> [Header]) -> IO [Header] + parseEarlyHintHeadersUntilFailure :: Int -> ([Header] -> [Header]) -> IO [Header] parseEarlyHintHeadersUntilFailure count front = do guardMaxNumberHeaders count line <- connectionReadLine mhl conn diff --git a/http-client/Network/HTTP/Client/Types.hs b/http-client/Network/HTTP/Client/Types.hs index 8473bf9e..3fb1d020 100644 --- a/http-client/Network/HTTP/Client/Types.hs +++ b/http-client/Network/HTTP/Client/Types.hs @@ -922,7 +922,7 @@ data StreamFileStatus = StreamFileStatus -- -- @since 0.7.14 newtype MaxHeaderLength = MaxHeaderLength - { unMaxHeaderLength :: Word + { unMaxHeaderLength :: Int } deriving (Eq, Show, Ord, Num, Enum, Bounded, T.Typeable) @@ -933,7 +933,7 @@ noMaxHeaderLength = maxBound -- -- @since 0.7.18 newtype MaxNumberHeaders = MaxNumberHeaders - { unMaxNumberHeaders :: Word + { unMaxNumberHeaders :: Int } deriving (Eq, Show, Ord, Num, Enum, Bounded, T.Typeable) From aa8930e865daf8a42d97c7f266c50e773d216dd1 Mon Sep 17 00:00:00 2001 From: Giorgio Marinelli Date: Mon, 16 Dec 2024 07:32:48 +0100 Subject: [PATCH 09/11] Update http-client changelog --- http-client/ChangeLog.md | 1 - 1 file changed, 1 deletion(-) diff --git a/http-client/ChangeLog.md b/http-client/ChangeLog.md index 3fad9bac..5f1405b7 100644 --- a/http-client/ChangeLog.md +++ b/http-client/ChangeLog.md @@ -3,7 +3,6 @@ ## 0.7.18 * Add the `managerSetMaxNumberHeaders` function to the `Client` module to configure `managerMaxNumberHeaders` in `ManagerSettings`. -* Use `Word` in the `MaxHeaderLength` and `MaxNumberHeaders` types. ## 0.7.17 From 749ed45c5a10af78b581752ee46e0a3a6f7c4858 Mon Sep 17 00:00:00 2001 From: Giorgio Marinelli Date: Mon, 16 Dec 2024 19:05:54 +0100 Subject: [PATCH 10/11] Revert "Use Word in MaxHeaderLength and MaxNumberHeaders" This reverts commit c3312b42871d1209b234df90c396aef6ab9e1d31. --- http-client/Network/HTTP/Client.hs | 4 ++-- http-client/Network/HTTP/Client/Body.hs | 2 +- http-client/Network/HTTP/Client/Connection.hs | 12 ++++++------ http-client/Network/HTTP/Client/Headers.hs | 13 ++++++------- http-client/Network/HTTP/Client/Manager.hs | 4 ++-- http-client/Network/HTTP/Client/Response.hs | 4 ++-- http-client/Network/HTTP/Client/Types.hs | 16 ++++------------ .../test-nonet/Network/HTTP/Client/BodySpec.hs | 16 ++++++++-------- .../Network/HTTP/Client/HeadersSpec.hs | 12 ++++++------ .../Network/HTTP/Client/ResponseSpec.hs | 2 +- http-conduit/test/main.hs | 5 +++-- 11 files changed, 41 insertions(+), 49 deletions(-) diff --git a/http-client/Network/HTTP/Client.hs b/http-client/Network/HTTP/Client.hs index d5cd9fe9..f0dbbc17 100644 --- a/http-client/Network/HTTP/Client.hs +++ b/http-client/Network/HTTP/Client.hs @@ -325,12 +325,12 @@ managerSetProxy po = managerSetInsecureProxy po . managerSetSecureProxy po -- @since 0.7.17 managerSetMaxHeaderLength :: Int -> ManagerSettings -> ManagerSettings managerSetMaxHeaderLength l manager = manager - { managerMaxHeaderLength = MaxHeaderLength l } + { managerMaxHeaderLength = Just $ MaxHeaderLength l } -- @since 0.7.18 managerSetMaxNumberHeaders :: Int -> ManagerSettings -> ManagerSettings managerSetMaxNumberHeaders n manager = manager - { managerMaxNumberHeaders = MaxNumberHeaders n } + { managerMaxNumberHeaders = Just $ MaxNumberHeaders n } -- $example1 -- = Example Usage diff --git a/http-client/Network/HTTP/Client/Body.hs b/http-client/Network/HTTP/Client/Body.hs index 09c6bc40..a44834cb 100644 --- a/http-client/Network/HTTP/Client/Body.hs +++ b/http-client/Network/HTTP/Client/Body.hs @@ -148,7 +148,7 @@ makeLengthReader cleanup count0 Connection {..} = do return bs makeChunkedReader - :: MaxHeaderLength + :: Maybe MaxHeaderLength -> IO () -- ^ cleanup -> Bool -- ^ raw -> Connection diff --git a/http-client/Network/HTTP/Client/Connection.hs b/http-client/Network/HTTP/Client/Connection.hs index 26db9638..a57da553 100644 --- a/http-client/Network/HTTP/Client/Connection.hs +++ b/http-client/Network/HTTP/Client/Connection.hs @@ -31,19 +31,19 @@ import Data.Function (fix) import Data.Maybe (listToMaybe) import Data.Word (Word8) -connectionReadLine :: MaxHeaderLength -> Connection -> IO ByteString +connectionReadLine :: Maybe MaxHeaderLength -> Connection -> IO ByteString connectionReadLine mhl conn = do bs <- connectionRead conn when (S.null bs) $ throwHttp IncompleteHeaders connectionReadLineWith mhl conn bs -- | Keep dropping input until a blank line is found. -connectionDropTillBlankLine :: MaxHeaderLength -> Connection -> IO () +connectionDropTillBlankLine :: Maybe MaxHeaderLength -> Connection -> IO () connectionDropTillBlankLine mhl conn = fix $ \loop -> do bs <- connectionReadLine mhl conn unless (S.null bs) loop -connectionReadLineWith :: MaxHeaderLength -> Connection -> ByteString -> IO ByteString +connectionReadLineWith :: Maybe MaxHeaderLength -> Connection -> ByteString -> IO ByteString connectionReadLineWith mhl conn bs0 = go bs0 id 0 where @@ -51,9 +51,9 @@ connectionReadLineWith mhl conn bs0 = case S.break (== charLF) bs of (_, "") -> do let total' = total + S.length bs - when (total > unMaxHeaderLength mhl) $ do - -- We reached the maximum length for an header field. - throwHttp OverlongHeaders + case fmap unMaxHeaderLength mhl of + Nothing -> pure () + Just n -> when (total' > n) $ throwHttp OverlongHeaders bs' <- connectionRead conn when (S.null bs') $ throwHttp IncompleteHeaders go bs' (front . (bs:)) total' diff --git a/http-client/Network/HTTP/Client/Headers.hs b/http-client/Network/HTTP/Client/Headers.hs index d1ba2bfe..f901702d 100644 --- a/http-client/Network/HTTP/Client/Headers.hs +++ b/http-client/Network/HTTP/Client/Headers.hs @@ -28,7 +28,7 @@ charColon = 58 charPeriod = 46 -parseStatusHeaders :: MaxHeaderLength -> MaxNumberHeaders -> Connection -> Maybe Int -> ([Header] -> IO ()) -> Maybe (IO ()) -> IO StatusHeaders +parseStatusHeaders :: Maybe MaxHeaderLength -> Maybe MaxNumberHeaders -> Connection -> Maybe Int -> ([Header] -> IO ()) -> Maybe (IO ()) -> IO StatusHeaders parseStatusHeaders mhl mnh conn timeout' onEarlyHintHeaders cont | Just k <- cont = getStatusExpectContinue k | otherwise = getStatus @@ -60,7 +60,7 @@ parseStatusHeaders mhl mnh conn timeout' onEarlyHintHeaders cont return $ Just $ StatusHeaders s' v' (earlyHeaders <> earlyHeaders') reqHeaders | otherwise -> (Just <$>) $ StatusHeaders s v mempty A.<$> parseHeaders 0 id - nextStatusLine :: MaxHeaderLength -> IO (Status, HttpVersion) + nextStatusLine :: Maybe MaxHeaderLength -> IO (Status, HttpVersion) nextStatusLine mhl = do -- Ensure that there is some data coming in. If not, we want to signal -- this as a connection problem and not a protocol problem. @@ -68,7 +68,7 @@ parseStatusHeaders mhl mnh conn timeout' onEarlyHintHeaders cont when (S.null bs) $ throwHttp NoResponseDataReceived connectionReadLineWith mhl conn bs >>= parseStatus mhl 3 - parseStatus :: MaxHeaderLength -> Int -> S.ByteString -> IO (Status, HttpVersion) + parseStatus :: Maybe MaxHeaderLength -> Int -> S.ByteString -> IO (Status, HttpVersion) parseStatus mhl i bs | S.null bs && i > 0 = connectionReadLine mhl conn >>= parseStatus mhl (i - 1) parseStatus _ _ bs = do let (ver, bs2) = S.break (== charSpace) bs @@ -92,10 +92,9 @@ parseStatusHeaders mhl mnh conn timeout' onEarlyHintHeaders cont _ -> Nothing guardMaxNumberHeaders :: Int -> IO () - guardMaxNumberHeaders count = - when (count >= unMaxNumberHeaders mnh) $ do - -- We reached the maximum number of header fields. - throwHttp TooManyHeaders + guardMaxNumberHeaders count = case fmap unMaxNumberHeaders mnh of + Nothing -> pure () + Just n -> when (count >= n) $ throwHttp TooManyHeaders parseHeaders :: Int -> ([Header] -> [Header]) -> IO [Header] parseHeaders count front = do diff --git a/http-client/Network/HTTP/Client/Manager.hs b/http-client/Network/HTTP/Client/Manager.hs index abe92ff6..6d0de348 100644 --- a/http-client/Network/HTTP/Client/Manager.hs +++ b/http-client/Network/HTTP/Client/Manager.hs @@ -92,8 +92,8 @@ defaultManagerSettings = ManagerSettings , managerModifyResponse = return , managerProxyInsecure = defaultProxy , managerProxySecure = defaultProxy - , managerMaxHeaderLength = 4096 - , managerMaxNumberHeaders = 100 + , managerMaxHeaderLength = Just $ MaxHeaderLength 4096 + , managerMaxNumberHeaders = Just $ MaxNumberHeaders 100 } -- | Create a 'Manager'. The @Manager@ will be shut down automatically via diff --git a/http-client/Network/HTTP/Client/Response.hs b/http-client/Network/HTTP/Client/Response.hs index 87de0298..52c3eb58 100644 --- a/http-client/Network/HTTP/Client/Response.hs +++ b/http-client/Network/HTTP/Client/Response.hs @@ -113,8 +113,8 @@ lbsResponse res = do { responseBody = L.fromChunks bss } -getResponse :: MaxHeaderLength - -> MaxNumberHeaders +getResponse :: Maybe MaxHeaderLength + -> Maybe MaxNumberHeaders -> Maybe Int -> Request -> Managed Connection diff --git a/http-client/Network/HTTP/Client/Types.hs b/http-client/Network/HTTP/Client/Types.hs index 3fb1d020..6647ed19 100644 --- a/http-client/Network/HTTP/Client/Types.hs +++ b/http-client/Network/HTTP/Client/Types.hs @@ -40,9 +40,7 @@ module Network.HTTP.Client.Types , ResponseTimeout (..) , ProxySecureMode (..) , MaxHeaderLength (..) - , noMaxHeaderLength , MaxNumberHeaders (..) - , noMaxNumberHeaders ) where import qualified Data.Typeable as T (Typeable) @@ -826,13 +824,13 @@ data ManagerSettings = ManagerSettings -- Default: respect the @proxy@ value on the @Request@ itself. -- -- Since 0.4.7 - , managerMaxHeaderLength :: MaxHeaderLength + , managerMaxHeaderLength :: Maybe MaxHeaderLength -- ^ Configure the maximum size, in bytes, of an HTTP header field. -- -- Default: 4096 -- -- @since 0.7.17 - , managerMaxNumberHeaders :: MaxNumberHeaders + , managerMaxNumberHeaders :: Maybe MaxNumberHeaders -- ^ Configure the maximum number of HTTP header fields. -- -- Default: 100 @@ -864,8 +862,8 @@ data Manager = Manager , mSetProxy :: Request -> Request , mModifyResponse :: Response BodyReader -> IO (Response BodyReader) -- ^ See 'managerProxy' - , mMaxHeaderLength :: MaxHeaderLength - , mMaxNumberHeaders :: MaxNumberHeaders + , mMaxHeaderLength :: Maybe MaxHeaderLength + , mMaxNumberHeaders :: Maybe MaxNumberHeaders } deriving T.Typeable @@ -926,9 +924,6 @@ newtype MaxHeaderLength = MaxHeaderLength } deriving (Eq, Show, Ord, Num, Enum, Bounded, T.Typeable) -noMaxHeaderLength :: MaxHeaderLength -noMaxHeaderLength = maxBound - -- | The maximum number of header fields. -- -- @since 0.7.18 @@ -936,6 +931,3 @@ newtype MaxNumberHeaders = MaxNumberHeaders { unMaxNumberHeaders :: Int } deriving (Eq, Show, Ord, Num, Enum, Bounded, T.Typeable) - -noMaxNumberHeaders :: MaxNumberHeaders -noMaxNumberHeaders = maxBound diff --git a/http-client/test-nonet/Network/HTTP/Client/BodySpec.hs b/http-client/test-nonet/Network/HTTP/Client/BodySpec.hs index 5bd28293..a2f2e613 100644 --- a/http-client/test-nonet/Network/HTTP/Client/BodySpec.hs +++ b/http-client/test-nonet/Network/HTTP/Client/BodySpec.hs @@ -22,7 +22,7 @@ spec = describe "BodySpec" $ do (conn, _, input) <- dummyConnection [ "5\r\nhello\r\n6\r\n world\r\n0\r\n\r\nnot consumed" ] - reader <- makeChunkedReader noMaxHeaderLength (return ()) False conn + reader <- makeChunkedReader Nothing (return ()) False conn body <- brConsume reader S.concat body `shouldBe` "hello world" input' <- input @@ -33,7 +33,7 @@ spec = describe "BodySpec" $ do (conn, _, input) <- dummyConnection [ "5\r\nhello\r\n6\r\n world\r\n0\r\ntrailers-are: ignored\r\nbut: consumed\r\n\r\nnot consumed" ] - reader <- makeChunkedReader noMaxHeaderLength (return ()) False conn + reader <- makeChunkedReader Nothing (return ()) False conn body <- brConsume reader S.concat body `shouldBe` "hello world" input' <- input @@ -43,7 +43,7 @@ spec = describe "BodySpec" $ do it "chunked, pieces" $ do (conn, _, input) <- dummyConnection $ map S.singleton $ S.unpack "5\r\nhello\r\n6\r\n world\r\n0\r\n\r\nnot consumed" - reader <- makeChunkedReader noMaxHeaderLength (return ()) False conn + reader <- makeChunkedReader Nothing (return ()) False conn body <- brConsume reader S.concat body `shouldBe` "hello world" input' <- input @@ -53,7 +53,7 @@ spec = describe "BodySpec" $ do it "chunked, pieces, with trailers" $ do (conn, _, input) <- dummyConnection $ map S.singleton $ S.unpack "5\r\nhello\r\n6\r\n world\r\n0\r\ntrailers-are: ignored\r\nbut: consumed\r\n\r\nnot consumed" - reader <- makeChunkedReader noMaxHeaderLength (return ()) False conn + reader <- makeChunkedReader Nothing (return ()) False conn body <- brConsume reader S.concat body `shouldBe` "hello world" input' <- input @@ -64,7 +64,7 @@ spec = describe "BodySpec" $ do (conn, _, input) <- dummyConnection [ "5\r\nhello\r\n6\r\n world\r\n0\r\n\r\nnot consumed" ] - reader <- makeChunkedReader noMaxHeaderLength (return ()) True conn + reader <- makeChunkedReader Nothing (return ()) True conn body <- brConsume reader S.concat body `shouldBe` "5\r\nhello\r\n6\r\n world\r\n0\r\n\r\n" input' <- input @@ -75,7 +75,7 @@ spec = describe "BodySpec" $ do (conn, _, input) <- dummyConnection [ "5\r\nhello\r\n6\r\n world\r\n0\r\ntrailers-are: returned\r\nin-raw: body\r\n\r\nnot consumed" ] - reader <- makeChunkedReader noMaxHeaderLength (return ()) True conn + reader <- makeChunkedReader Nothing (return ()) True conn body <- brConsume reader S.concat body `shouldBe` "5\r\nhello\r\n6\r\n world\r\n0\r\ntrailers-are: returned\r\nin-raw: body\r\n\r\n" input' <- input @@ -85,7 +85,7 @@ spec = describe "BodySpec" $ do it "chunked, pieces, raw" $ do (conn, _, input) <- dummyConnection $ map S.singleton $ S.unpack "5\r\nhello\r\n6\r\n world\r\n0\r\n\r\nnot consumed" - reader <- makeChunkedReader noMaxHeaderLength (return ()) True conn + reader <- makeChunkedReader Nothing (return ()) True conn body <- brConsume reader S.concat body `shouldBe` "5\r\nhello\r\n6\r\n world\r\n0\r\n\r\n" input' <- input @@ -95,7 +95,7 @@ spec = describe "BodySpec" $ do it "chunked, pieces, raw, with trailers" $ do (conn, _, input) <- dummyConnection $ map S.singleton $ S.unpack "5\r\nhello\r\n6\r\n world\r\n0\r\ntrailers-are: returned\r\nin-raw: body\r\n\r\nnot consumed" - reader <- makeChunkedReader noMaxHeaderLength (return ()) True conn + reader <- makeChunkedReader Nothing (return ()) True conn body <- brConsume reader S.concat body `shouldBe` "5\r\nhello\r\n6\r\n world\r\n0\r\ntrailers-are: returned\r\nin-raw: body\r\n\r\n" input' <- input diff --git a/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs b/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs index 955505d3..eb0b04c1 100644 --- a/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs +++ b/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs @@ -23,7 +23,7 @@ spec = describe "HeadersSpec" $ do , "\nignored" ] (connection, _, _) <- dummyConnection input - statusHeaders <- parseStatusHeaders noMaxHeaderLength noMaxNumberHeaders connection Nothing (\_ -> return ()) Nothing + statusHeaders <- parseStatusHeaders Nothing Nothing connection Nothing (\_ -> return ()) Nothing statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) mempty [ ("foo", "bar") , ("baz", "bin") @@ -37,7 +37,7 @@ spec = describe "HeadersSpec" $ do ] (conn, out, _) <- dummyConnection input let sendBody = connectionWrite conn "data" - statusHeaders <- parseStatusHeaders noMaxHeaderLength noMaxNumberHeaders conn Nothing (\_ -> return ()) (Just sendBody) + statusHeaders <- parseStatusHeaders Nothing Nothing conn Nothing (\_ -> return ()) (Just sendBody) statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [] [ ("foo", "bar") ] out >>= (`shouldBe` ["data"]) @@ -47,7 +47,7 @@ spec = describe "HeadersSpec" $ do ] (conn, out, _) <- dummyConnection input let sendBody = connectionWrite conn "data" - statusHeaders <- parseStatusHeaders noMaxHeaderLength noMaxNumberHeaders conn Nothing (\_ -> return ()) (Just sendBody) + statusHeaders <- parseStatusHeaders Nothing Nothing conn Nothing (\_ -> return ()) (Just sendBody) statusHeaders `shouldBe` StatusHeaders status417 (HttpVersion 1 1) [] [] out >>= (`shouldBe` []) @@ -59,7 +59,7 @@ spec = describe "HeadersSpec" $ do , "result" ] (conn, out, inp) <- dummyConnection input - statusHeaders <- parseStatusHeaders noMaxHeaderLength noMaxNumberHeaders conn Nothing (\_ -> return ()) Nothing + statusHeaders <- parseStatusHeaders Nothing Nothing conn Nothing (\_ -> return ()) Nothing statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [] [ ("foo", "bar") ] out >>= (`shouldBe` []) inp >>= (`shouldBe` ["result"]) @@ -78,7 +78,7 @@ spec = describe "HeadersSpec" $ do callbackResults :: MVar (Seq.Seq [Header]) <- newMVar mempty let onEarlyHintHeader h = modifyMVar_ callbackResults (return . (Seq.|> h)) - statusHeaders <- parseStatusHeaders noMaxHeaderLength noMaxNumberHeaders conn Nothing onEarlyHintHeader Nothing + statusHeaders <- parseStatusHeaders Nothing Nothing conn Nothing onEarlyHintHeader Nothing statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [("Link", "") , ("Link", "") @@ -110,7 +110,7 @@ spec = describe "HeadersSpec" $ do callbackResults :: MVar (Seq.Seq [Header]) <- newMVar mempty let onEarlyHintHeader h = modifyMVar_ callbackResults (return . (Seq.|> h)) - statusHeaders <- parseStatusHeaders noMaxHeaderLength noMaxNumberHeaders conn Nothing onEarlyHintHeader Nothing + statusHeaders <- parseStatusHeaders Nothing Nothing conn Nothing onEarlyHintHeader Nothing statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [("Link", "") , ("Link", "") diff --git a/http-client/test-nonet/Network/HTTP/Client/ResponseSpec.hs b/http-client/test-nonet/Network/HTTP/Client/ResponseSpec.hs index 3728fd4f..cdb8d8ec 100644 --- a/http-client/test-nonet/Network/HTTP/Client/ResponseSpec.hs +++ b/http-client/test-nonet/Network/HTTP/Client/ResponseSpec.hs @@ -16,7 +16,7 @@ main = hspec spec spec :: Spec spec = describe "ResponseSpec" $ do - let getResponse' conn = getResponse noMaxHeaderLength noMaxNumberHeaders Nothing req (dummyManaged conn) Nothing + let getResponse' conn = getResponse Nothing Nothing Nothing req (dummyManaged conn) Nothing req = parseRequest_ "http://localhost" it "basic" $ do (conn, _, _) <- dummyConnection diff --git a/http-conduit/test/main.hs b/http-conduit/test/main.hs index 43144ee2..2a760748 100644 --- a/http-conduit/test/main.hs +++ b/http-conduit/test/main.hs @@ -56,6 +56,7 @@ import qualified Data.Aeson as A import qualified Network.HTTP.Simple as Simple import Data.Monoid (mempty) import Control.Monad.Trans.Resource (runResourceT) +import Data.Maybe (fromJust) past :: UTCTime past = UTCTime (ModifiedJulianDay 56200) (secondsToDiffTime 0) @@ -542,7 +543,7 @@ tooManyHeaderFields :: (Int -> IO ()) -> IO () tooManyHeaderFields = withCApp $ \app' -> runConduit $ src .| appSink app' where - limit = fromEnum (managerMaxNumberHeaders defaultManagerSettings) + limit = fromEnum (fromJust $ managerMaxNumberHeaders defaultManagerSettings) src = sourceList $ "HTTP/1.0 200 OK\r\n" : replicate limit "foo: bar\r\n" notTooManyHeaderFields :: (Int -> IO ()) -> IO () @@ -550,7 +551,7 @@ notTooManyHeaderFields = withCApp $ \app' -> do runConduit $ appSource app' .| CL.drop 1 runConduit $ src .| appSink app' where - limit = fromEnum (managerMaxNumberHeaders defaultManagerSettings) - 1 + limit = fromEnum (fromJust $ managerMaxNumberHeaders defaultManagerSettings) - 1 src = sourceList $ ["HTTP/1.0 200 OK\r\n"] <> replicate limit "foo: bar\r\n" <> ["\r\n"] redir :: (Int -> IO ()) -> IO () From 1e82860af40aa6af77442289a7f8f3422b5dee91 Mon Sep 17 00:00:00 2001 From: Giorgio Marinelli Date: Wed, 18 Dec 2024 14:13:00 +0100 Subject: [PATCH 11/11] Move tests and rename error --- http-client/Network/HTTP/Client/Headers.hs | 2 +- http-client/Network/HTTP/Client/Types.hs | 8 ++--- .../test-nonet/Network/HTTP/ClientSpec.hs | 24 ++++++++++++++ http-conduit/Network/HTTP/Simple.hs | 1 + http-conduit/test/main.hs | 31 +------------------ 5 files changed, 31 insertions(+), 35 deletions(-) diff --git a/http-client/Network/HTTP/Client/Headers.hs b/http-client/Network/HTTP/Client/Headers.hs index f901702d..7c436bd2 100644 --- a/http-client/Network/HTTP/Client/Headers.hs +++ b/http-client/Network/HTTP/Client/Headers.hs @@ -94,7 +94,7 @@ parseStatusHeaders mhl mnh conn timeout' onEarlyHintHeaders cont guardMaxNumberHeaders :: Int -> IO () guardMaxNumberHeaders count = case fmap unMaxNumberHeaders mnh of Nothing -> pure () - Just n -> when (count >= n) $ throwHttp TooManyHeaders + Just n -> when (count >= n) $ throwHttp TooManyHeaderFields parseHeaders :: Int -> ([Header] -> [Header]) -> IO [Header] parseHeaders count front = do diff --git a/http-client/Network/HTTP/Client/Types.hs b/http-client/Network/HTTP/Client/Types.hs index 6647ed19..13bc5ac9 100644 --- a/http-client/Network/HTTP/Client/Types.hs +++ b/http-client/Network/HTTP/Client/Types.hs @@ -149,12 +149,12 @@ data HttpExceptionContent -- -- @since 0.5.0 | OverlongHeaders - -- ^ Too many total bytes in a single header field were - -- returned by the server. + -- ^ Too many total bytes in the HTTP header were returned + -- by the server. -- -- @since 0.5.0 - | TooManyHeaders - -- ^ Too many header fields were returned by the server, + | TooManyHeaderFields + -- ^ Too many HTTP header fields were returned by the server. -- -- @since 0.7.18 | ResponseTimeout diff --git a/http-client/test-nonet/Network/HTTP/ClientSpec.hs b/http-client/test-nonet/Network/HTTP/ClientSpec.hs index 15bbc30d..6793baa4 100644 --- a/http-client/test-nonet/Network/HTTP/ClientSpec.hs +++ b/http-client/test-nonet/Network/HTTP/ClientSpec.hs @@ -31,6 +31,9 @@ notWindows _ = return () notWindows x = x #endif +crlf :: S.ByteString +crlf = "\r\n" + main :: IO () main = hspec spec @@ -323,3 +326,24 @@ spec = describe "Client" $ do case parseRequest "https://o_O:18446744072699450606" of Left _ -> pure () :: IO () Right req -> error $ "Invalid request: " ++ show req + + it "too many header fields" $ do + let message = S.concat $ + ["HTTP/1.1 200 OK", crlf] <> replicate 120 ("foo: bar" <> crlf) <> [crlf, "body"] + + serveWith message $ \port -> do + man <- newManager $ managerSetMaxNumberHeaders 120 defaultManagerSettings + req <- parseUrlThrow $ "http://127.0.0.1:" ++ show port + httpLbs req man `shouldThrow` \e -> case e of + HttpExceptionRequest _ TooManyHeaderFields -> True + _otherwise -> False + + it "not too many header fields" $ do + let message = S.concat $ + ["HTTP/1.1 200 OK", crlf] <> replicate 120 ("foo: bar" <> crlf) <> [crlf, "body"] + + serveWith message $ \port -> do + man <- newManager $ managerSetMaxNumberHeaders 121 defaultManagerSettings + req <- parseUrlThrow $ "http://127.0.0.1:" ++ show port + res <- httpLbs req man + responseBody res `shouldBe` "body" diff --git a/http-conduit/Network/HTTP/Simple.hs b/http-conduit/Network/HTTP/Simple.hs index e46005d6..25672f87 100644 --- a/http-conduit/Network/HTTP/Simple.hs +++ b/http-conduit/Network/HTTP/Simple.hs @@ -110,6 +110,7 @@ import qualified Data.Aeson as A import qualified Data.Traversable as T import Control.Exception (throw, throwIO, Exception) +import Data.Monoid import Data.Typeable (Typeable) import qualified Data.Conduit as C import Data.Conduit (runConduit, (.|), ConduitM) diff --git a/http-conduit/test/main.hs b/http-conduit/test/main.hs index 2a760748..632f5f81 100644 --- a/http-conduit/test/main.hs +++ b/http-conduit/test/main.hs @@ -8,8 +8,7 @@ import qualified Data.ByteString.Lazy.Char8 as L8 import Test.HUnit import Network.Wai hiding (requestBody) import Network.Wai.Conduit (responseSource, sourceRequestBody) -import Network.HTTP.Client (streamFile, defaultManagerSettings) -import Network.HTTP.Client.Internal (managerMaxNumberHeaders) +import Network.HTTP.Client (streamFile) import System.IO.Temp (withSystemTempFile) import qualified Network.Wai as Wai import Network.Wai.Handler.Warp (runSettings, defaultSettings, setPort, setBeforeMainLoop, Settings, setTimeout) @@ -56,7 +55,6 @@ import qualified Data.Aeson as A import qualified Network.HTTP.Simple as Simple import Data.Monoid (mempty) import Control.Monad.Trans.Resource (runResourceT) -import Data.Maybe (fromJust) past :: UTCTime past = UTCTime (ModifiedJulianDay 56200) (secondsToDiffTime 0) @@ -263,18 +261,6 @@ main = do let Just req1 = parseUrlThrow $ "http://127.0.0.1:" ++ show port _ <- httpLbs req1 manager return () - it "too many header fields" $ tooManyHeaderFields $ \port -> do - manager <- newManager tlsManagerSettings - let Just req1 = parseUrlThrow $ "http://127.0.0.1:" ++ show port - res1 <- try $ runResourceT $ http req1 manager - case res1 of - Left e -> show (e :: SomeException) @?= show (HttpExceptionRequest req1 TooManyHeaders) - _ -> error "Shouldn't have worked" - it "not too many header fields" $ notTooManyHeaderFields $ \port -> do - manager <- newManager tlsManagerSettings - let Just req1 = parseUrlThrow $ "http://127.0.0.1:" ++ show port - _ <- httpLbs req1 manager - return () describe "redirects" $ do it "doesn't double escape" $ redir $ \port -> do manager <- newManager tlsManagerSettings @@ -539,21 +525,6 @@ notOverLongHeaders = withCApp $ \app' -> do where src = sourceList $ [S.concat $ "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 16384\r\n\r\n" : ( take 16384 $ repeat "x")] -tooManyHeaderFields :: (Int -> IO ()) -> IO () -tooManyHeaderFields = - withCApp $ \app' -> runConduit $ src .| appSink app' - where - limit = fromEnum (fromJust $ managerMaxNumberHeaders defaultManagerSettings) - src = sourceList $ "HTTP/1.0 200 OK\r\n" : replicate limit "foo: bar\r\n" - -notTooManyHeaderFields :: (Int -> IO ()) -> IO () -notTooManyHeaderFields = withCApp $ \app' -> do - runConduit $ appSource app' .| CL.drop 1 - runConduit $ src .| appSink app' - where - limit = fromEnum (fromJust $ managerMaxNumberHeaders defaultManagerSettings) - 1 - src = sourceList $ ["HTTP/1.0 200 OK\r\n"] <> replicate limit "foo: bar\r\n" <> ["\r\n"] - redir :: (Int -> IO ()) -> IO () redir = withApp' redirApp