Skip to content

Commit

Permalink
xml tuple
Browse files Browse the repository at this point in the history
  • Loading branch information
@jph00
jph00 committed May 15, 2024
1 parent bed9cba commit 89ffc8b
Show file tree
Hide file tree
Showing 5 changed files with 71 additions and 23 deletions.
1 change: 1 addition & 0 deletions fastcore/_modidx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,7 @@
'fastcore.parallel.parallel_gen': ('parallel.html#parallel_gen', 'fastcore/parallel.py'),
'fastcore.parallel.parallelable': ('parallel.html#parallelable', 'fastcore/parallel.py'),
'fastcore.parallel.run_procs': ('parallel.html#run_procs', 'fastcore/parallel.py'),
'fastcore.parallel.startproc': ('parallel.html#startproc', 'fastcore/parallel.py'),
'fastcore.parallel.startthread': ('parallel.html#startthread', 'fastcore/parallel.py'),
'fastcore.parallel.threaded': ('parallel.html#threaded', 'fastcore/parallel.py')},
'fastcore.script': { 'fastcore.script.Param': ('script.html#param', 'fastcore/script.py'),
Expand Down
29 changes: 17 additions & 12 deletions fastcore/parallel.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/03a_parallel.ipynb.

# %% auto 0
__all__ = ['threaded', 'startthread', 'parallelable', 'ThreadPoolExecutor', 'ProcessPoolExecutor', 'parallel', 'add_one',
'run_procs', 'parallel_gen']
__all__ = ['threaded', 'startthread', 'startproc', 'parallelable', 'ThreadPoolExecutor', 'ProcessPoolExecutor', 'parallel',
'add_one', 'run_procs', 'parallel_gen']

# %% ../nbs/03a_parallel.ipynb 1
from .imports import *
Expand Down Expand Up @@ -38,9 +38,14 @@ def _f(*args, **kwargs):
# %% ../nbs/03a_parallel.ipynb 6
def startthread(f):
"Like `threaded`, but start thread immediately"
threaded(f)()
return threaded(f)()

# %% ../nbs/03a_parallel.ipynb 8
def startproc(f):
"Like `threaded(True)`, but start Process immediately"
return threaded(True)(f)()

# %% ../nbs/03a_parallel.ipynb 10
def _call(lock, pause, n, g, item):
l = False
if pause:
Expand All @@ -51,7 +56,7 @@ def _call(lock, pause, n, g, item):
if l: lock.release()
return g(item)

# %% ../nbs/03a_parallel.ipynb 9
# %% ../nbs/03a_parallel.ipynb 11
def parallelable(param_name, num_workers, f=None):
f_in_main = f == None or sys.modules[f.__module__].__name__ == "__main__"
if sys.platform == "win32" and IN_NOTEBOOK and num_workers > 0 and f_in_main:
Expand All @@ -60,7 +65,7 @@ def parallelable(param_name, num_workers, f=None):
return False
return True

# %% ../nbs/03a_parallel.ipynb 10
# %% ../nbs/03a_parallel.ipynb 12
class ThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):
"Same as Python's ThreadPoolExecutor, except can pass `max_workers==0` for serial execution"
def __init__(self, max_workers=defaults.cpus, on_exc=print, pause=0, **kwargs):
Expand All @@ -78,7 +83,7 @@ def map(self, f, items, *args, timeout=None, chunksize=1, **kwargs):
try: return super().map(_g, items, timeout=timeout, chunksize=chunksize)
except Exception as e: self.on_exc(e)

# %% ../nbs/03a_parallel.ipynb 12
# %% ../nbs/03a_parallel.ipynb 14
@delegates()
class ProcessPoolExecutor(concurrent.futures.ProcessPoolExecutor):
"Same as Python's ProcessPoolExecutor, except can pass `max_workers==0` for serial execution"
Expand All @@ -101,11 +106,11 @@ def map(self, f, items, *args, timeout=None, chunksize=1, **kwargs):
try: return super().map(_g, items, timeout=timeout, chunksize=chunksize)
except Exception as e: self.on_exc(e)

# %% ../nbs/03a_parallel.ipynb 14
# %% ../nbs/03a_parallel.ipynb 16
try: from fastprogress import progress_bar
except: progress_bar = None

# %% ../nbs/03a_parallel.ipynb 15
# %% ../nbs/03a_parallel.ipynb 17
def parallel(f, items, *args, n_workers=defaults.cpus, total=None, progress=None, pause=0,
method=None, threadpool=False, timeout=None, chunksize=1, **kwargs):
"Applies `func` in parallel to `items`, using `n_workers`"
Expand All @@ -122,28 +127,28 @@ def parallel(f, items, *args, n_workers=defaults.cpus, total=None, progress=None
r = progress_bar(r, total=total, leave=False)
return L(r)

# %% ../nbs/03a_parallel.ipynb 16
# %% ../nbs/03a_parallel.ipynb 18
def add_one(x, a=1):
# this import is necessary for multiprocessing in notebook on windows
import random
time.sleep(random.random()/80)
return x+a

# %% ../nbs/03a_parallel.ipynb 22
# %% ../nbs/03a_parallel.ipynb 24
def run_procs(f, f_done, args):
"Call `f` for each item in `args` in parallel, yielding `f_done`"
processes = L(args).map(Process, args=arg0, target=f)
for o in processes: o.start()
yield from f_done()
processes.map(Self.join())

# %% ../nbs/03a_parallel.ipynb 23
# %% ../nbs/03a_parallel.ipynb 25
def _f_pg(obj, queue, batch, start_idx):
for i,b in enumerate(obj(batch)): queue.put((start_idx+i,b))

def _done_pg(queue, items): return (queue.get() for _ in items)

# %% ../nbs/03a_parallel.ipynb 24
# %% ../nbs/03a_parallel.ipynb 26
def parallel_gen(cls, items, n_workers=defaults.cpus, **kwargs):
"Instantiate `cls` in `n_workers` procs & call each on a subset of `items` in parallel."
if not parallelable('n_workers', n_workers): n_workers = 0
Expand Down
11 changes: 6 additions & 5 deletions fastcore/xml.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,19 @@
from .utils import *

import types
from dataclasses import dataclass
from functools import partial
from html import escape

# %% ../nbs/11_xml.ipynb 4
def _attrmap(o):
return dict(cls='class', klass='class', fr='for').get(o, o)
o = dict(htmlClass='class', cls='class', klass='class', fr='for', htmlFor='for').get(o, o)
return o.lstrip('_').replace('_', '-')

# %% ../nbs/11_xml.ipynb 5
def xt(tag:str, *c, **kw):
"Create an XML tag structure `[tag,children,attrs]` for `toxml()`"
if len(c)==1 and isinstance(c[0], types.GeneratorType): c = tuple(c[0])
kw = {_attrmap(k.lstrip('_')):str(v) for k,v in kw.items()}
kw = {_attrmap(k):str(v) for k,v in kw.items()}
return [tag.lower(),c,kw]

# %% ../nbs/11_xml.ipynb 6
Expand All @@ -45,6 +45,7 @@ def xt(tag:str, *c, **kw):
# %% ../nbs/11_xml.ipynb 9
def to_xml(elm, lvl=0):
"Convert `xt` element tree into an XML string"
if isinstance(elm, tuple): return '\n'.join(to_xml(o) for o in elm)
if hasattr(elm, '__xt__'): elm = elm.__xt__()
sp = ' ' * lvl
if not isinstance(elm, list):
Expand All @@ -54,11 +55,11 @@ def to_xml(elm, lvl=0):
tag,cs,attrs = elm
stag = tag
if attrs:
sattrs = (f'{k}="{escape(str(v))}"' for k,v in attrs.items())
sattrs = (f'{k}="{escape(str(v), quote=False)}"' for k,v in attrs.items())
stag += ' ' + ' '.join(sattrs)

if not cs: return f'{sp}<{stag}></{tag}>\n'
res = f'{sp}<{stag}>\n'
res += ''.join(to_xml(c, lvl+2) for c in cs)
res += ''.join(to_xml(c, lvl=lvl+2) for c in cs)
res += f'{sp}</{tag}>\n'
return res
42 changes: 41 additions & 1 deletion nbs/03a_parallel.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@
"#|export\n",
"def startthread(f):\n",
" \"Like `threaded`, but start thread immediately\"\n",
" threaded(f)()"
" return threaded(f)()"
]
},
{
Expand Down Expand Up @@ -144,6 +144,46 @@
"time.sleep(0.1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#|export\n",
"def startproc(f):\n",
" \"Like `threaded(True)`, but start Process immediately\"\n",
" return threaded(True)(f)()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"first\n",
"second\n"
]
}
],
"source": [
"@startproc\n",
"def _():\n",
" time.sleep(0.05)\n",
" print(\"second\")\n",
"\n",
"@startproc\n",
"def _():\n",
" time.sleep(0.01)\n",
" print(\"first\")\n",
"\n",
"time.sleep(0.1)"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down
11 changes: 6 additions & 5 deletions nbs/11_xml.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
"from fastcore.utils import *\n",
"\n",
"import types\n",
"from dataclasses import dataclass\n",
"from functools import partial\n",
"from html import escape"
]
Expand All @@ -55,7 +54,8 @@
"source": [
"#| export\n",
"def _attrmap(o):\n",
" return dict(cls='class', klass='class', fr='for').get(o, o)"
" o = dict(htmlClass='class', cls='class', klass='class', fr='for', htmlFor='for').get(o, o)\n",
" return o.lstrip('_').replace('_', '-')"
]
},
{
Expand All @@ -69,7 +69,7 @@
"def xt(tag:str, *c, **kw):\n",
" \"Create an XML tag structure `[tag,children,attrs]` for `toxml()`\"\n",
" if len(c)==1 and isinstance(c[0], types.GeneratorType): c = tuple(c[0])\n",
" kw = {_attrmap(k.lstrip('_')):str(v) for k,v in kw.items()}\n",
" kw = {_attrmap(k):str(v) for k,v in kw.items()}\n",
" return [tag.lower(),c,kw]"
]
},
Expand Down Expand Up @@ -147,6 +147,7 @@
"#| export\n",
"def to_xml(elm, lvl=0):\n",
" \"Convert `xt` element tree into an XML string\"\n",
" if isinstance(elm, tuple): return '\\n'.join(to_xml(o) for o in elm)\n",
" if hasattr(elm, '__xt__'): elm = elm.__xt__()\n",
" sp = ' ' * lvl\n",
" if not isinstance(elm, list):\n",
Expand All @@ -156,12 +157,12 @@
" tag,cs,attrs = elm\n",
" stag = tag\n",
" if attrs:\n",
" sattrs = (f'{k}=\"{escape(str(v))}\"' for k,v in attrs.items())\n",
" sattrs = (f'{k}=\"{escape(str(v), quote=False)}\"' for k,v in attrs.items())\n",
" stag += ' ' + ' '.join(sattrs)\n",
" \n",
" if not cs: return f'{sp}<{stag}></{tag}>\\n'\n",
" res = f'{sp}<{stag}>\\n'\n",
" res += ''.join(to_xml(c, lvl+2) for c in cs)\n",
" res += ''.join(to_xml(c, lvl=lvl+2) for c in cs)\n",
" res += f'{sp}</{tag}>\\n'\n",
" return res"
]
Expand Down

0 comments on commit 89ffc8b

Please sign in to comment.