Skip to content

Commit

Permalink
fixes #562
Browse files Browse the repository at this point in the history
  • Loading branch information
@jph00
jph00 committed May 30, 2024
1 parent 954233c commit 985ee19
Show file tree
Hide file tree
Showing 5 changed files with 139 additions and 33 deletions.
3 changes: 3 additions & 0 deletions fastcore/_modidx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -377,8 +377,11 @@
'fastcore.net.Request.summary': ('net.html#request.summary', 'fastcore/net.py'),
'fastcore.net._socket_det': ('net.html#_socket_det', 'fastcore/net.py'),
'fastcore.net.do_request': ('net.html#do_request', 'fastcore/net.py'),
'fastcore.net.http_response': ('net.html#http_response', 'fastcore/net.py'),
'fastcore.net.recv_once': ('net.html#recv_once', 'fastcore/net.py'),
'fastcore.net.start_client': ('net.html#start_client', 'fastcore/net.py'),
'fastcore.net.start_server': ('net.html#start_server', 'fastcore/net.py'),
'fastcore.net.tobytes': ('net.html#tobytes', 'fastcore/net.py'),
'fastcore.net.urlcheck': ('net.html#urlcheck', 'fastcore/net.py'),
'fastcore.net.urlclean': ('net.html#urlclean', 'fastcore/net.py'),
'fastcore.net.urldest': ('net.html#urldest', 'fastcore/net.py'),
Expand Down
79 changes: 52 additions & 27 deletions fastcore/net.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@
# %% auto 0
__all__ = ['url_default_headers', 'ExceptionsHTTP', 'urlquote', 'urlwrap', 'HTTP4xxClientError', 'HTTP5xxServerError',
'urlopener', 'urlopen', 'urlread', 'urljson', 'urlcheck', 'urlclean', 'urlretrieve', 'urldest', 'urlsave',
'urlvalid', 'urlrequest', 'urlsend', 'do_request', 'start_server', 'start_client', 'HTTP400BadRequestError',
'HTTP401UnauthorizedError', 'HTTP402PaymentRequiredError', 'HTTP403ForbiddenError', 'HTTP404NotFoundError',
'urlvalid', 'urlrequest', 'urlsend', 'do_request', 'start_server', 'start_client', 'tobytes',
'http_response', 'recv_once', 'HTTP400BadRequestError', 'HTTP401UnauthorizedError',
'HTTP402PaymentRequiredError', 'HTTP403ForbiddenError', 'HTTP404NotFoundError',
'HTTP405MethodNotAllowedError', 'HTTP406NotAcceptableError', 'HTTP407ProxyAuthRequiredError',
'HTTP408RequestTimeoutError', 'HTTP409ConflictError', 'HTTP410GoneError', 'HTTP411LengthRequiredError',
'HTTP412PreconditionFailedError', 'HTTP413PayloadTooLargeError', 'HTTP414URITooLongError',
Expand All @@ -27,7 +28,7 @@
from urllib.parse import urlencode,urlparse,urlunparse
from http.client import InvalidURL

# %% ../nbs/03b_net.ipynb 4
# %% ../nbs/03b_net.ipynb 5
url_default_headers = {
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
Expand All @@ -41,7 +42,7 @@
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
}

# %% ../nbs/03b_net.ipynb 5
# %% ../nbs/03b_net.ipynb 6
def urlquote(url):
"Update url's path with `urllib.parse.quote`"
subdelims = "!$&'()*+,;="
Expand All @@ -52,31 +53,31 @@ def urlquote(url):
for i in range(3,6): p[i] = urllib.parse.quote(p[i], safe=safe)
return urlunparse(p)

# %% ../nbs/03b_net.ipynb 8
# %% ../nbs/03b_net.ipynb 9
def urlwrap(url, data=None, headers=None):
"Wrap `url` in a urllib `Request` with `urlquote`"
return url if isinstance(url,Request) else Request(urlquote(url), data=data, headers=headers or {})

# %% ../nbs/03b_net.ipynb 9
# %% ../nbs/03b_net.ipynb 10
ExceptionsHTTP = {}

# %% ../nbs/03b_net.ipynb 10
# %% ../nbs/03b_net.ipynb 11
class HTTP4xxClientError(HTTPError):
"Base class for client exceptions (code 4xx) from `url*` functions"
pass

# %% ../nbs/03b_net.ipynb 11
# %% ../nbs/03b_net.ipynb 12
class HTTP5xxServerError(HTTPError):
"Base class for server exceptions (code 5xx) from `url*` functions"
pass

# %% ../nbs/03b_net.ipynb 14
# %% ../nbs/03b_net.ipynb 15
def urlopener():
_opener = urllib.request.build_opener()
_opener.addheaders = list(url_default_headers.items())
return _opener

# %% ../nbs/03b_net.ipynb 15
# %% ../nbs/03b_net.ipynb 16
# install_opener(_opener)

_httperrors = (
Expand All @@ -95,10 +96,10 @@ def _init(self, url, hdrs, fp, msg=msg, code=code): HTTP4xxClientError.__init__(
cls = type(nm, (HTTP4xxClientError,), {'__init__':_init})
globals()[nm] = ExceptionsHTTP[code] = cls

# %% ../nbs/03b_net.ipynb 16
# %% ../nbs/03b_net.ipynb 17
_all_ = ['HTTP400BadRequestError', 'HTTP401UnauthorizedError', 'HTTP402PaymentRequiredError', 'HTTP403ForbiddenError', 'HTTP404NotFoundError', 'HTTP405MethodNotAllowedError', 'HTTP406NotAcceptableError', 'HTTP407ProxyAuthRequiredError', 'HTTP408RequestTimeoutError', 'HTTP409ConflictError', 'HTTP410GoneError', 'HTTP411LengthRequiredError', 'HTTP412PreconditionFailedError', 'HTTP413PayloadTooLargeError', 'HTTP414URITooLongError', 'HTTP415UnsupportedMediaTypeError', 'HTTP416RangeNotSatisfiableError', 'HTTP417ExpectationFailedError', 'HTTP418AmAteapotError', 'HTTP421MisdirectedRequestError', 'HTTP422UnprocessableEntityError', 'HTTP423LockedError', 'HTTP424FailedDependencyError', 'HTTP425TooEarlyError', 'HTTP426UpgradeRequiredError', 'HTTP428PreconditionRequiredError', 'HTTP429TooManyRequestsError', 'HTTP431HeaderFieldsTooLargeError', 'HTTP451LegalReasonsError']

# %% ../nbs/03b_net.ipynb 17
# %% ../nbs/03b_net.ipynb 18
def urlopen(url, data=None, headers=None, timeout=None, **kwargs):
"Like `urllib.request.urlopen`, but first `urlwrap` the `url`, and encode `data`"
if kwargs and not data: data=kwargs
Expand All @@ -110,7 +111,7 @@ def urlopen(url, data=None, headers=None, timeout=None, **kwargs):
e.msg += f"\n====Error Body====\n{e.read().decode(errors='ignore')}"
raise

# %% ../nbs/03b_net.ipynb 20
# %% ../nbs/03b_net.ipynb 21
def urlread(url, data=None, headers=None, decode=True, return_json=False, return_headers=False, timeout=None, **kwargs):
"Retrieve `url`, using `data` dict or `kwargs` to `POST` if present"
try:
Expand All @@ -123,13 +124,13 @@ def urlread(url, data=None, headers=None, decode=True, return_json=False, return
if return_json: res = loads(res)
return (res,dict(hdrs)) if return_headers else res

# %% ../nbs/03b_net.ipynb 21
# %% ../nbs/03b_net.ipynb 22
def urljson(url, data=None, timeout=None):
"Retrieve `url` and decode json"
res = urlread(url, data=data, timeout=timeout)
return json.loads(res) if res else {}

# %% ../nbs/03b_net.ipynb 23
# %% ../nbs/03b_net.ipynb 24
def urlcheck(url, headers=None, timeout=10):
if not url: return True
try:
Expand All @@ -138,12 +139,12 @@ def urlcheck(url, headers=None, timeout=10):
except socket.timeout: return False
except InvalidURL: return False

# %% ../nbs/03b_net.ipynb 24
# %% ../nbs/03b_net.ipynb 25
def urlclean(url):
"Remove fragment, params, and querystring from `url` if present"
return urlunparse(urlparse(str(url))[:3]+('','',''))

# %% ../nbs/03b_net.ipynb 26
# %% ../nbs/03b_net.ipynb 27
def urlretrieve(url, filename=None, reporthook=None, data=None, headers=None, timeout=None):
"Same as `urllib.request.urlretrieve` but also works with `Request` objects"
with contextlib.closing(urlopen(url, data, headers=headers, timeout=timeout)) as fp:
Expand All @@ -169,43 +170,43 @@ def urlretrieve(url, filename=None, reporthook=None, data=None, headers=None, ti
raise ContentTooShortError(f"retrieval incomplete: got only {read} out of {size} bytes", headers)
return filename,headers

# %% ../nbs/03b_net.ipynb 27
# %% ../nbs/03b_net.ipynb 28
def urldest(url, dest=None):
name = urlclean(Path(url).name)
if dest is None: dest = name
dest = Path(dest)
return dest/name if dest.is_dir() else dest

# %% ../nbs/03b_net.ipynb 28
# %% ../nbs/03b_net.ipynb 29
def urlsave(url, dest=None, reporthook=None, headers=None, timeout=None):
"Retrieve `url` and save based on its name"
dest = urldest(url, dest)
dest.parent.mkdir(parents=True, exist_ok=True)
nm,msg = urlretrieve(url, dest, reporthook, headers=headers, timeout=timeout)
return nm

# %% ../nbs/03b_net.ipynb 30
# %% ../nbs/03b_net.ipynb 31
def urlvalid(x):
"Test if `x` is a valid URL"
return all (getattrs(urlparse(str(x)), 'scheme', 'netloc'))

# %% ../nbs/03b_net.ipynb 32
# %% ../nbs/03b_net.ipynb 33
def urlrequest(url, verb, headers=None, route=None, query=None, data=None, json_data=True):
"`Request` for `url` with optional route params replaced by `route`, plus `query` string, and post `data`"
if route: url = url.format(**route)
if query: url += '?' + urlencode(query)
if isinstance(data,dict): data = (json.dumps if json_data else urlencode)(data).encode('ascii')
return Request(url, headers=headers or {}, data=data or None, method=verb.upper())

# %% ../nbs/03b_net.ipynb 35
# %% ../nbs/03b_net.ipynb 36
@patch
def summary(self:Request, skip=None)->dict:
"Summary containing full_url, headers, method, and data, removing `skip` from headers"
res = L('full_url','method','data').map_dict(partial(getattr,self))
res['headers'] = {k:v for k,v in self.headers.items() if k not in listify(skip)}
return res

# %% ../nbs/03b_net.ipynb 37
# %% ../nbs/03b_net.ipynb 38
def urlsend(url, verb, headers=None, route=None, query=None, data=None, json_data=True,
return_json=True, return_headers=False, debug=None, timeout=None):
"Send request with `urlrequest`, converting result to json if `return_json`"
Expand All @@ -217,7 +218,7 @@ def urlsend(url, verb, headers=None, route=None, query=None, data=None, json_dat

return urlread(req, return_json=return_json, return_headers=return_headers, timeout=timeout)

# %% ../nbs/03b_net.ipynb 38
# %% ../nbs/03b_net.ipynb 39
def do_request(url, post=False, headers=None, **data):
"Call GET or json-encoded POST on `url`, depending on `post`"
if data:
Expand All @@ -227,13 +228,13 @@ def do_request(url, post=False, headers=None, **data):
data = None
return urljson(Request(url, headers=headers, data=data or None))

# %% ../nbs/03b_net.ipynb 39
# %% ../nbs/03b_net.ipynb 41
def _socket_det(port,host,dgram):
if isinstance(port,int): family,addr = socket.AF_INET,(host or socket.gethostname(),port)
else: family,addr = socket.AF_UNIX,port
return family,addr,(socket.SOCK_STREAM,socket.SOCK_DGRAM)[dgram]

# %% ../nbs/03b_net.ipynb 40
# %% ../nbs/03b_net.ipynb 42
def start_server(port, host=None, dgram=False, reuse_addr=True, n_queue=None):
"Create a `socket` server on `port`, with optional `host`, of type `dgram`"
listen_args = [n_queue] if n_queue else []
Expand All @@ -247,10 +248,34 @@ def start_server(port, host=None, dgram=False, reuse_addr=True, n_queue=None):
s.listen(*listen_args)
return s

# %% ../nbs/03b_net.ipynb 42
# %% ../nbs/03b_net.ipynb 44
def start_client(port, host=None, dgram=False):
"Create a `socket` client on `port`, with optional `host`, of type `dgram`"
family,addr,typ = _socket_det(port,host,dgram)
s = socket.socket(family, typ)
s.connect(addr)
return s

# %% ../nbs/03b_net.ipynb 45
def tobytes(s:str)->bytes:
"Convert `s` into HTTP-ready bytes format"
return s.replace('\n', '\r\n').encode('utf-8')

# %% ../nbs/03b_net.ipynb 47
def http_response(body=None, status=200, hdrs=None, **kwargs):
"Create an HTTP-ready response, adding `kwargs` to `hdrs`"
kwargs = {k.replace('_','-'):v for k,v in kwargs.items()}
hdrs = hdrs or {}
hdrs = {**hdrs, **kwargs}
status_line = f"HTTP/1.1 {status} OK\n"
if body: hdrs['Content-Length'] = len(body)
headers = ''.join([f"{k}: {v}\n" for k, v in hdrs.items()])
return tobytes(status_line+headers+"\n" + str(body))

# %% ../nbs/03b_net.ipynb 49
@threaded
def recv_once(d:dict, host:str='localhost', port:int=8000):
"Spawn a thread to receive a single HTTP request and store in `d['r']`"
conn,addr = start_server(port,host).accept()
d['r'] = conn.recv(1024)
conn.sendall(http_response(d['r']))
3 changes: 2 additions & 1 deletion fastcore/xml.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@

# %% ../nbs/11_xml.ipynb 4
def _attrmap(o):
o = dict(htmlClass='class', cls='class', klass='class', fr='for', htmlFor='for').get(o, o)
o = dict(htmlClass='class', cls='class', _class='class', klass='class',
_for='for', fr='for', htmlFor='for').get(o, o)
return o.lstrip('_').replace('_', '-')

# %% ../nbs/11_xml.ipynb 5
Expand Down
78 changes: 78 additions & 0 deletions nbs/03b_net.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,13 @@
"> Network, HTTP, and URL functions"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## URLs"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -642,6 +649,13 @@
" return urljson(Request(url, headers=headers, data=data or None))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Basic client/server"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -698,6 +712,70 @@
" return s"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#|export\n",
"def tobytes(s:str)->bytes:\n",
" \"Convert `s` into HTTP-ready bytes format\"\n",
" return s.replace('\\n', '\\r\\n').encode('utf-8')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"test_eq(tobytes('foo\\nbar'), b'foo\\r\\nbar')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#|export\n",
"def http_response(body=None, status=200, hdrs=None, **kwargs):\n",
" \"Create an HTTP-ready response, adding `kwargs` to `hdrs`\"\n",
" kwargs = {k.replace('_','-'):v for k,v in kwargs.items()}\n",
" hdrs = hdrs or {}\n",
" hdrs = {**hdrs, **kwargs}\n",
" status_line = f\"HTTP/1.1 {status} OK\\n\"\n",
" if body: hdrs['Content-Length'] = len(body)\n",
" headers = ''.join([f\"{k}: {v}\\n\" for k, v in hdrs.items()])\n",
" return tobytes(status_line+headers+\"\\n\" + str(body))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"exp = b'HTTP/1.1 200 OK\\r\\nUser-Agent: me\\r\\nContent-Length: 4\\r\\n\\r\\nbody'\n",
"test_eq(http_response('body', 200, User_Agent='me'), exp)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#|export\n",
"@threaded\n",
"def recv_once(d:dict, host:str='localhost', port:int=8000):\n",
" \"Spawn a thread to receive a single HTTP request and store in `d['r']`\"\n",
" conn,addr = start_server(port,host).accept()\n",
" d['r'] = conn.recv(1024)\n",
" conn.sendall(http_response(d['r']))"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand Down
9 changes: 4 additions & 5 deletions nbs/11_xml.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@
"source": [
"#| export\n",
"def _attrmap(o):\n",
" o = dict(htmlClass='class', cls='class', klass='class', fr='for', htmlFor='for').get(o, o)\n",
" o = dict(htmlClass='class', cls='class', _class='class', klass='class',\n",
" _for='for', fr='for', htmlFor='for').get(o, o)\n",
" return o.lstrip('_').replace('_', '-')"
]
},
Expand Down Expand Up @@ -139,7 +140,7 @@
" (['head', (['title', ('Some page',), {}],), {}],\n",
" ['body',\n",
" (['div',\n",
" (['p', ('Some text',), {}],\n",
" ('Some text',\n",
" ['input', (), {'name': 'me'}],\n",
" ['img', (), {'src': 'filename'}]),\n",
" {'class': 'myclass'}],),\n",
Expand All @@ -151,7 +152,7 @@
"source": [
"samp = Html(\n",
" Head(Title('Some page')),\n",
" Body(Div(P('Some text'), Input(name='me'), Img(src=\"filename\"), klass='myclass'))\n",
" Body(Div('Some text', Input(name='me'), Img(src=\"filename\"), klass='myclass'))\n",
")\n",
"pprint(samp)"
]
Expand Down Expand Up @@ -261,9 +262,7 @@
" </head>\n",
" <body>\n",
" <div class=\"myclass\">\n",
" <p>\n",
"Some text\n",
" </p>\n",
" <input name=\"me\">\n",
" <img src=\"filename\">\n",
" </div>\n",
Expand Down

0 comments on commit 985ee19

Please sign in to comment.