From 1f5fc6338b0c5eb88c6716344dbd45aea27b4a64 Mon Sep 17 00:00:00 2001 From: Maqbool ur Rahim Khan Date: Thu, 30 May 2019 13:48:24 +0200 Subject: [PATCH 01/12] Home for NLE constructed! --- butler/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 butler/.gitkeep diff --git a/butler/.gitkeep b/butler/.gitkeep new file mode 100644 index 000000000..e69de29bb From 1226a5ab41632ab52a0079fca2eae91a1f3f61c1 Mon Sep 17 00:00:00 2001 From: Nandeesh Date: Tue, 4 Jun 2019 12:29:09 +0200 Subject: [PATCH 02/12] First commit to NLE. --- butler/requirements.txt | 83 ++++++++++++++++++++++ butler/{.gitkeep => upb/__init__.py} | 0 butler/upb/ida/__init__.py | 0 butler/upb/ida/nle/__init__.py | 0 butler/upb/ida/nle/frquencytransformer.pk | Bin 0 -> 1156 bytes butler/upb/ida/nle/model.pkl | Bin 0 -> 3084 bytes butler/upb/ida/nle/server.py | 52 ++++++++++++++ butler/upb/ida/nle/tokenizer.pk | Bin 0 -> 1884 bytes butler/upb/ida/nle/train.py | 33 +++++++++ 9 files changed, 168 insertions(+) create mode 100644 butler/requirements.txt rename butler/{.gitkeep => upb/__init__.py} (100%) create mode 100644 butler/upb/ida/__init__.py create mode 100644 butler/upb/ida/nle/__init__.py create mode 100644 butler/upb/ida/nle/frquencytransformer.pk create mode 100644 butler/upb/ida/nle/model.pkl create mode 100644 butler/upb/ida/nle/server.py create mode 100644 butler/upb/ida/nle/tokenizer.pk create mode 100644 butler/upb/ida/nle/train.py diff --git a/butler/requirements.txt b/butler/requirements.txt new file mode 100644 index 000000000..7936f3bc6 --- /dev/null +++ b/butler/requirements.txt @@ -0,0 +1,83 @@ +apturl==0.5.2 +asn1crypto==0.24.0 +attrs==19.1.0 +Brlapi==0.6.6 +certifi==2018.1.18 +chardet==3.0.4 +Click==7.0 +command-not-found==0.3 +cryptography==2.1.4 +cupshelpers==1.0 +cycler==0.10.0 +decorator==4.4.0 +defer==1.0.6 +distro-info===0.18ubuntu0.18.04.1 +Flask==1.0.3 +Flask-Cors==3.0.7 +httplib2==0.9.2 +idna==2.6 +ipython-genutils==0.2.0 +itsdangerous==1.1.0 +Jinja2==2.10.1 +joblib==0.13.2 +jsonschema==3.0.1 +jupyter-core==4.4.0 +keyring==10.6.0 +keyrings.alt==3.0 +kiwisolver==1.1.0 +language-selector==0.1 +launchpadlib==1.10.6 +lazr.restfulclient==0.13.5 +lazr.uri==1.0.3 +louis==3.5.0 +macaroonbakery==1.1.3 +Mako==1.0.7 +MarkupSafe==1.0 +matplotlib==3.1.0 +nbformat==4.4.0 +netifaces==0.10.4 +numpy==1.16.4 +oauth==1.0.1 +olefile==0.45.1 +pandas==0.24.2 +pexpect==4.2.1 +Pillow==5.1.0 +plotly==3.10.0 +protobuf==3.0.0 +pycairo==1.16.2 +pycrypto==2.6.1 +pycups==1.9.73 +pygobject==3.26.1 +pymacaroons==0.13.0 +PyNaCl==1.1.2 +pyparsing==2.4.0 +pyRFC3339==1.0 +pyrsistent==0.15.2 +python-apt==1.6.4 +python-dateutil==2.6.1 +python-debian==0.1.32 +pytz==2018.3 +pyxdg==0.25 +PyYAML==3.12 +reportlab==3.4.0 +requests==2.18.4 +requests-unixsocket==0.1.5 +retrying==1.3.3 +scikit-learn==0.21.2 +scipy==1.3.0 +SecretStorage==2.3.1 +simplejson==3.13.2 +six==1.11.0 +system-service==0.3 +systemd-python==234 +traitlets==4.3.2 +ubuntu-drivers-common==0.0.0 +ufw==0.36 +unattended-upgrades==0.1 +urllib3==1.22 +usb-creator==0.3.3 +virtualenv==16.6.0 +wadllib==1.3.2 +Werkzeug==0.15.4 +xkit==0.0.0 +zope.interface==4.3.2 diff --git a/butler/.gitkeep b/butler/upb/__init__.py similarity index 100% rename from butler/.gitkeep rename to butler/upb/__init__.py diff --git a/butler/upb/ida/__init__.py b/butler/upb/ida/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/butler/upb/ida/nle/__init__.py b/butler/upb/ida/nle/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/butler/upb/ida/nle/frquencytransformer.pk b/butler/upb/ida/nle/frquencytransformer.pk new file mode 100644 index 0000000000000000000000000000000000000000..5fa3b33bdb19ce7383f0cccb0d485ae628d7f408 GIT binary patch literal 1156 zcmd6n%Wl&^6o#Eko78QZv`uJ5Oj1zZJ6p1C4K6`x5neqJp8TnPjbe}gVVUE(E zgnNwYw96SWx!raYj$`?>Vb>d{j5uzi&03VnARWAh&>4gaT2zuG2crV}AgaZ1 z;JH-CSuk~>$@uEF+HG$6k?9L8`oi`vh)MhXy@_OAS&meBPrrqf&Zig_J zDyGY1L~jw!?5+$m_47RP5&;b5i}ye6SCeynGY;{_FA0{J*{j wbr0VU&eJcW*E)lB;Xa2W!!tB-H1+ZSMSnq=i$4`8`bX7ws$(Rc%r-uTvHG~opRu@Urek#mJC=g|*%|LE-dkYFnv{nyg z*A{SIBt?-Wn3=L$J#LzzyjBJS;XW1Sw)t*>!XT*t5jj!|z$$^#1R_%mQiH4l4Dt(Y zOi7^~11cf}n$61sgAEdcv=XRIU`UD~l#oJJ2ZkDrMnj@OW>7F-m}PBFnFZ1$JQO@u zcpQ&m27b5sG54L*(eT*e;V}0sfrrcGxW{=2`yZ|z$GhMu>}US}0fsXm%0fg!*7uuP zJzh6s2GNdiNB_hz4mfVC3B>uw9pUgzxHtLH2i;W-l+-QR;@^lFgoh+jLsOeT>; z_r6+?Uwa-$(dQzOv|P%Ix9q!vUv1487QOupF60qT-g?e>v-<*nU$0wgnliVU$4H0b zPdCb=_kYoa=Q(m8_9;7fU!gzJ4;0hRcLSTV*ZCdyE=O0vWqzKkMq{fC>j;$ zTj-N8DI@bz(rM8-kUlOXp*oLl3evUAYFyhiPpU_{ikQs0@|t%1!S0-s*A{f(|EO~t zwlzHr%0v3|A2{!|O%?wgkxne8|6Mrs;(hn5tn+Lbs zKED-|e>Y9_VA1-3kFZV)QxVnFbt%Zt<&?LEz2C<3o!jeQSx0y9SlGX?t{5(!Kdoc) ziMxEA4;v?q*UY(#gXbCWf`u3Y^U(Hkd6vL`9*@NgQqTfRn&r%UlQ&`P z!@23{bb*||7y6X$YCD!48eCVX6T@Rj66<(F&K_P`7j9e@T&J`Cdgbti7~7kzZ;d~$ z5bZ;(e#U_vr|K+`Y^SO(^pHZu$EiALH?+ZckJG97NjviYa!Hdf%#qled`OpOPj$sw7xhmPa5 N6DMdVf=sVP`ycmIm}dY0 literal 0 HcmV?d00001 diff --git a/butler/upb/ida/nle/server.py b/butler/upb/ida/nle/server.py new file mode 100644 index 000000000..53833f392 --- /dev/null +++ b/butler/upb/ida/nle/server.py @@ -0,0 +1,52 @@ +import pickle +import flask +from flask import request +from flask_cors import CORS + +app = flask.Flask(__name__) +CORS(app) + +# Extract the dumped model from pickle file +model = pickle.load(open("model.pkl", "rb")) +# extract the dumped tokenizer for pre-processing the user message. +tokenizer = pickle.load(open('tokenizer.pk', 'rb')) + +# Extract the token frequency transformer from dumped pickle file. +frequency_transformer = pickle.load(open('frquencytransformer.pk', 'rb')) + + +# Api to classify the user message. +@app.route('/predict', methods=['GET']) +def index(): + # Getting the user message from url parameter. + query = request.args.get('msg') + if query is None: + return flask.jsonify({ + 'error': 'No query parameter' + }) + + # creating a response object + # storing the model's prediction in the object + response = {} + + # tokenizing the user input using the existing tokenizer. + test_counts = tokenizer.transform([query]) + # converting the tokens into frequencies. + test_frequencies = frequency_transformer.transform(test_counts) + cls = model.predict(test_frequencies) + + # set the reply based on the class + if cls == 'greet': + response['predictions'] = 'Hello' + elif cls == 'help': + response['predictions'] = 'I can load datasets for you. \n I can visualise data too.' + elif cls == 'dataset': + response['predictions'] = 'Dataset loaded.' + elif cls == 'barchart': + response['predictions'] = 'Barchart drawn.' + + # returning the response object as json + return flask.jsonify(response) + + +app.run() diff --git a/butler/upb/ida/nle/tokenizer.pk b/butler/upb/ida/nle/tokenizer.pk new file mode 100644 index 0000000000000000000000000000000000000000..eb189009f6761e50e80119c0820e5741c8e83cc8 GIT binary patch literal 1884 zcmYk7`Fa~Q6vf9)m(<<&eW#F6YT6J=fI@%(g%}J7CsTq*fVo9nm8_$wm{W} zec#Kz@B98RJOXb3l5YIb^M^l6zoUDvMx#3?`oe6oCAf@Bb-`6GMIfe?aYk^BBP^17U2z^MBy0b4EXar&x_(TSLpK>nEzoCD-kQq_5008#9>m4;s)k9 zW3K9ZM=;-CY*0-YWKv0)NG$N@o2g96C{*b8m&^{*fQO-pmSJJVElQOjNPlFS=rqG_Zj(tf;tJ+k85x)v@x>=s$)V(QFjk*;VC&E7zK4pJ|5G z6@QWO#SO{Zf#h*RU`@GDE@-TEIi`}3SMwH^U96)V)hM>3^)v3_)l@eX*bv5fJM9*u zSREKHVxzyp#H>ftq89ugAi$;>f2jdMrh0`3Q4O2N{~kW;xiZcH*b-LrsHLJfD`r9! zvGoYHox*m89nN}g+?~g;iw?ahTCD`rS=GC7vU0FNqbN*LZ>9MfXUYp%j~PC3Ie1f;gZ-2o#Es)Kjn5gra2qu#@Fj8EI;`-O#@7trxNS6|DS>av zu$;Ju?=-$=_`ww!O7p1`_>nC3Q)PvpG=66I#l?vE6vwY**g)LFZyLWd{E_=MYB2lV Z45oz4%*PUcy4d?lL;FiZ_$#lL{sXea^n(BZ literal 0 HcmV?d00001 diff --git a/butler/upb/ida/nle/train.py b/butler/upb/ida/nle/train.py new file mode 100644 index 000000000..04be65c46 --- /dev/null +++ b/butler/upb/ida/nle/train.py @@ -0,0 +1,33 @@ +from sklearn.feature_extraction.text import CountVectorizer +from sklearn.feature_extraction.text import TfidfTransformer +from sklearn.naive_bayes import MultinomialNB +import pickle + +training_data = ["Hello", "Hello how are you", "Hi", "Hey, whats up?", "Hey", "What you can do for me?", + "How can you help me?", "What can you do?", "What are you capable of doing?", "Load $city database", + "Open $city dataset", "Please load $city dataset", "can you please load $city dataset", + "Load dataset $city data for me", "Can you draw me bar chart for this table?", + "Draw me a bar chart for random table", "Go ahead and give me a bar chart", + "Create me bar chart of nava database"] +# training_target = ["0", "0", "0", "0", "0", "1", "1", "1", "1", "2", "2", "2", "2", "2", "3", "3", "3", "3"] +training_classes = ["greet", "greet", "greet", "greet", "greet", "help", "help", "help", "help", "dataset", "dataset", + "dataset", "dataset", "dataset", "barchart", "barchart", "barchart", "barchart"] +classes = ['greet', 'help', 'dataset', 'barchart'] + +tokenizer = CountVectorizer() +training_counts = tokenizer.fit_transform(training_data) +frequency_transformer = TfidfTransformer() +training_frequencies = frequency_transformer.fit_transform(training_counts) +classifier = MultinomialNB().partial_fit(training_frequencies, training_classes, classes=classes) + +pickle.dump(classifier, open("model.pkl", "wb")) +with open('tokenizer.pk', 'wb') as fin: + pickle.dump(tokenizer, fin) +with open('frquencytransformer.pk', 'wb') as fin: + pickle.dump(frequency_transformer, fin) + + +test_counts = tokenizer.transform(['load movie data']) +test_frequencies = frequency_transformer.transform(test_counts) +ans = classifier.predict(test_frequencies) +print(ans) From 19645139aad6570a148379e38b99edabaa6041ad Mon Sep 17 00:00:00 2001 From: Maqbool ur Rahim Khan Date: Sun, 23 Jun 2019 22:41:22 +0200 Subject: [PATCH 03/12] NLE reinitialized with fasttext --- .gitignore | 3 + butler/.gitignore | 127 ++++++++++++++++++++++ butler/data.db | 20 ++++ butler/main.py | 10 ++ butler/requirements.txt | 85 +-------------- butler/upb/__init__.py | 0 butler/upb/ida/__init__.py | 0 butler/upb/ida/nle/__init__.py | 0 butler/upb/ida/nle/frquencytransformer.pk | Bin 1156 -> 0 bytes butler/upb/ida/nle/model.pkl | Bin 3084 -> 0 bytes butler/upb/ida/nle/server.py | 52 --------- butler/upb/ida/nle/tokenizer.pk | Bin 1884 -> 0 bytes butler/upb/ida/nle/train.py | 33 ------ 13 files changed, 163 insertions(+), 167 deletions(-) create mode 100644 butler/.gitignore create mode 100644 butler/data.db create mode 100644 butler/main.py delete mode 100644 butler/upb/__init__.py delete mode 100644 butler/upb/ida/__init__.py delete mode 100644 butler/upb/ida/nle/__init__.py delete mode 100644 butler/upb/ida/nle/frquencytransformer.pk delete mode 100644 butler/upb/ida/nle/model.pkl delete mode 100644 butler/upb/ida/nle/server.py delete mode 100644 butler/upb/ida/nle/tokenizer.pk delete mode 100644 butler/upb/ida/nle/train.py diff --git a/.gitignore b/.gitignore index 80a6f2aa5..bfdd82b4d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ out/ *.iws *.iml + +# mac +.DS_Store diff --git a/butler/.gitignore b/butler/.gitignore new file mode 100644 index 000000000..ee623cd6b --- /dev/null +++ b/butler/.gitignore @@ -0,0 +1,127 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# model +*.bin \ No newline at end of file diff --git a/butler/data.db b/butler/data.db new file mode 100644 index 000000000..e9f33c3a6 --- /dev/null +++ b/butler/data.db @@ -0,0 +1,20 @@ +__label__greet Hello +__label__greet Hello dear +__label__greet Hi +__label__greet Hey, whats up? +__label__greet Hey +__label__help What you can do for me? +__label__help How can you help me? +__label__help What can you do? +__label__help What are you capable of doing? + +__label__dataset Load <> database +__label__dataset Open <> dataset +__label__dataset Please load <> dataset +__label__dataset can you please load <> dataset +__label__dataset Load dataset <> data for me + +__label__barchart Can you draw me bar chart for this table? +__label__barchart Draw me a bar chart for random table +__label__barchart Go ahead and give me a bar chart +__label__barchart Create me bar chart of nava database diff --git a/butler/main.py b/butler/main.py new file mode 100644 index 000000000..f05d003c3 --- /dev/null +++ b/butler/main.py @@ -0,0 +1,10 @@ +import fasttext + +corpus = 'data.db' + +classifier = fasttext.supervised(corpus, 'model_chat', label_prefix='__label__') + +msg = input('Enter text> ') +while msg != 'q': + print(classifier.predict([msg])) + msg = input('Enter text> ') diff --git a/butler/requirements.txt b/butler/requirements.txt index 7936f3bc6..5a891839b 100644 --- a/butler/requirements.txt +++ b/butler/requirements.txt @@ -1,83 +1,4 @@ -apturl==0.5.2 -asn1crypto==0.24.0 -attrs==19.1.0 -Brlapi==0.6.6 -certifi==2018.1.18 -chardet==3.0.4 -Click==7.0 -command-not-found==0.3 -cryptography==2.1.4 -cupshelpers==1.0 -cycler==0.10.0 -decorator==4.4.0 -defer==1.0.6 -distro-info===0.18ubuntu0.18.04.1 -Flask==1.0.3 -Flask-Cors==3.0.7 -httplib2==0.9.2 -idna==2.6 -ipython-genutils==0.2.0 -itsdangerous==1.1.0 -Jinja2==2.10.1 -joblib==0.13.2 -jsonschema==3.0.1 -jupyter-core==4.4.0 -keyring==10.6.0 -keyrings.alt==3.0 -kiwisolver==1.1.0 -language-selector==0.1 -launchpadlib==1.10.6 -lazr.restfulclient==0.13.5 -lazr.uri==1.0.3 -louis==3.5.0 -macaroonbakery==1.1.3 -Mako==1.0.7 -MarkupSafe==1.0 -matplotlib==3.1.0 -nbformat==4.4.0 -netifaces==0.10.4 +Cython==0.29.10 +fasttext==0.8.3 +future==0.17.1 numpy==1.16.4 -oauth==1.0.1 -olefile==0.45.1 -pandas==0.24.2 -pexpect==4.2.1 -Pillow==5.1.0 -plotly==3.10.0 -protobuf==3.0.0 -pycairo==1.16.2 -pycrypto==2.6.1 -pycups==1.9.73 -pygobject==3.26.1 -pymacaroons==0.13.0 -PyNaCl==1.1.2 -pyparsing==2.4.0 -pyRFC3339==1.0 -pyrsistent==0.15.2 -python-apt==1.6.4 -python-dateutil==2.6.1 -python-debian==0.1.32 -pytz==2018.3 -pyxdg==0.25 -PyYAML==3.12 -reportlab==3.4.0 -requests==2.18.4 -requests-unixsocket==0.1.5 -retrying==1.3.3 -scikit-learn==0.21.2 -scipy==1.3.0 -SecretStorage==2.3.1 -simplejson==3.13.2 -six==1.11.0 -system-service==0.3 -systemd-python==234 -traitlets==4.3.2 -ubuntu-drivers-common==0.0.0 -ufw==0.36 -unattended-upgrades==0.1 -urllib3==1.22 -usb-creator==0.3.3 -virtualenv==16.6.0 -wadllib==1.3.2 -Werkzeug==0.15.4 -xkit==0.0.0 -zope.interface==4.3.2 diff --git a/butler/upb/__init__.py b/butler/upb/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/butler/upb/ida/__init__.py b/butler/upb/ida/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/butler/upb/ida/nle/__init__.py b/butler/upb/ida/nle/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/butler/upb/ida/nle/frquencytransformer.pk b/butler/upb/ida/nle/frquencytransformer.pk deleted file mode 100644 index 5fa3b33bdb19ce7383f0cccb0d485ae628d7f408..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1156 zcmd6n%Wl&^6o#Eko78QZv`uJ5Oj1zZJ6p1C4K6`x5neqJp8TnPjbe}gVVUE(E zgnNwYw96SWx!raYj$`?>Vb>d{j5uzi&03VnARWAh&>4gaT2zuG2crV}AgaZ1 z;JH-CSuk~>$@uEF+HG$6k?9L8`oi`vh)MhXy@_OAS&meBPrrqf&Zig_J zDyGY1L~jw!?5+$m_47RP5&;b5i}ye6SCeynGY;{_FA0{J*{j wbr0VU&eJcW*E)lB;Xa2W!!tB-H1+ZSMSnq=i$4`8`bX7ws$(Rc%r-uTvHG~opRu@Urek#mJC=g|*%|LE-dkYFnv{nyg z*A{SIBt?-Wn3=L$J#LzzyjBJS;XW1Sw)t*>!XT*t5jj!|z$$^#1R_%mQiH4l4Dt(Y zOi7^~11cf}n$61sgAEdcv=XRIU`UD~l#oJJ2ZkDrMnj@OW>7F-m}PBFnFZ1$JQO@u zcpQ&m27b5sG54L*(eT*e;V}0sfrrcGxW{=2`yZ|z$GhMu>}US}0fsXm%0fg!*7uuP zJzh6s2GNdiNB_hz4mfVC3B>uw9pUgzxHtLH2i;W-l+-QR;@^lFgoh+jLsOeT>; z_r6+?Uwa-$(dQzOv|P%Ix9q!vUv1487QOupF60qT-g?e>v-<*nU$0wgnliVU$4H0b zPdCb=_kYoa=Q(m8_9;7fU!gzJ4;0hRcLSTV*ZCdyE=O0vWqzKkMq{fC>j;$ zTj-N8DI@bz(rM8-kUlOXp*oLl3evUAYFyhiPpU_{ikQs0@|t%1!S0-s*A{f(|EO~t zwlzHr%0v3|A2{!|O%?wgkxne8|6Mrs;(hn5tn+Lbs zKED-|e>Y9_VA1-3kFZV)QxVnFbt%Zt<&?LEz2C<3o!jeQSx0y9SlGX?t{5(!Kdoc) ziMxEA4;v?q*UY(#gXbCWf`u3Y^U(Hkd6vL`9*@NgQqTfRn&r%UlQ&`P z!@23{bb*||7y6X$YCD!48eCVX6T@Rj66<(F&K_P`7j9e@T&J`Cdgbti7~7kzZ;d~$ z5bZ;(e#U_vr|K+`Y^SO(^pHZu$EiALH?+ZckJG97NjviYa!Hdf%#qled`OpOPj$sw7xhmPa5 N6DMdVf=sVP`ycmIm}dY0 diff --git a/butler/upb/ida/nle/server.py b/butler/upb/ida/nle/server.py deleted file mode 100644 index 53833f392..000000000 --- a/butler/upb/ida/nle/server.py +++ /dev/null @@ -1,52 +0,0 @@ -import pickle -import flask -from flask import request -from flask_cors import CORS - -app = flask.Flask(__name__) -CORS(app) - -# Extract the dumped model from pickle file -model = pickle.load(open("model.pkl", "rb")) -# extract the dumped tokenizer for pre-processing the user message. -tokenizer = pickle.load(open('tokenizer.pk', 'rb')) - -# Extract the token frequency transformer from dumped pickle file. -frequency_transformer = pickle.load(open('frquencytransformer.pk', 'rb')) - - -# Api to classify the user message. -@app.route('/predict', methods=['GET']) -def index(): - # Getting the user message from url parameter. - query = request.args.get('msg') - if query is None: - return flask.jsonify({ - 'error': 'No query parameter' - }) - - # creating a response object - # storing the model's prediction in the object - response = {} - - # tokenizing the user input using the existing tokenizer. - test_counts = tokenizer.transform([query]) - # converting the tokens into frequencies. - test_frequencies = frequency_transformer.transform(test_counts) - cls = model.predict(test_frequencies) - - # set the reply based on the class - if cls == 'greet': - response['predictions'] = 'Hello' - elif cls == 'help': - response['predictions'] = 'I can load datasets for you. \n I can visualise data too.' - elif cls == 'dataset': - response['predictions'] = 'Dataset loaded.' - elif cls == 'barchart': - response['predictions'] = 'Barchart drawn.' - - # returning the response object as json - return flask.jsonify(response) - - -app.run() diff --git a/butler/upb/ida/nle/tokenizer.pk b/butler/upb/ida/nle/tokenizer.pk deleted file mode 100644 index eb189009f6761e50e80119c0820e5741c8e83cc8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1884 zcmYk7`Fa~Q6vf9)m(<<&eW#F6YT6J=fI@%(g%}J7CsTq*fVo9nm8_$wm{W} zec#Kz@B98RJOXb3l5YIb^M^l6zoUDvMx#3?`oe6oCAf@Bb-`6GMIfe?aYk^BBP^17U2z^MBy0b4EXar&x_(TSLpK>nEzoCD-kQq_5008#9>m4;s)k9 zW3K9ZM=;-CY*0-YWKv0)NG$N@o2g96C{*b8m&^{*fQO-pmSJJVElQOjNPlFS=rqG_Zj(tf;tJ+k85x)v@x>=s$)V(QFjk*;VC&E7zK4pJ|5G z6@QWO#SO{Zf#h*RU`@GDE@-TEIi`}3SMwH^U96)V)hM>3^)v3_)l@eX*bv5fJM9*u zSREKHVxzyp#H>ftq89ugAi$;>f2jdMrh0`3Q4O2N{~kW;xiZcH*b-LrsHLJfD`r9! zvGoYHox*m89nN}g+?~g;iw?ahTCD`rS=GC7vU0FNqbN*LZ>9MfXUYp%j~PC3Ie1f;gZ-2o#Es)Kjn5gra2qu#@Fj8EI;`-O#@7trxNS6|DS>av zu$;Ju?=-$=_`ww!O7p1`_>nC3Q)PvpG=66I#l?vE6vwY**g)LFZyLWd{E_=MYB2lV Z45oz4%*PUcy4d?lL;FiZ_$#lL{sXea^n(BZ diff --git a/butler/upb/ida/nle/train.py b/butler/upb/ida/nle/train.py deleted file mode 100644 index 04be65c46..000000000 --- a/butler/upb/ida/nle/train.py +++ /dev/null @@ -1,33 +0,0 @@ -from sklearn.feature_extraction.text import CountVectorizer -from sklearn.feature_extraction.text import TfidfTransformer -from sklearn.naive_bayes import MultinomialNB -import pickle - -training_data = ["Hello", "Hello how are you", "Hi", "Hey, whats up?", "Hey", "What you can do for me?", - "How can you help me?", "What can you do?", "What are you capable of doing?", "Load $city database", - "Open $city dataset", "Please load $city dataset", "can you please load $city dataset", - "Load dataset $city data for me", "Can you draw me bar chart for this table?", - "Draw me a bar chart for random table", "Go ahead and give me a bar chart", - "Create me bar chart of nava database"] -# training_target = ["0", "0", "0", "0", "0", "1", "1", "1", "1", "2", "2", "2", "2", "2", "3", "3", "3", "3"] -training_classes = ["greet", "greet", "greet", "greet", "greet", "help", "help", "help", "help", "dataset", "dataset", - "dataset", "dataset", "dataset", "barchart", "barchart", "barchart", "barchart"] -classes = ['greet', 'help', 'dataset', 'barchart'] - -tokenizer = CountVectorizer() -training_counts = tokenizer.fit_transform(training_data) -frequency_transformer = TfidfTransformer() -training_frequencies = frequency_transformer.fit_transform(training_counts) -classifier = MultinomialNB().partial_fit(training_frequencies, training_classes, classes=classes) - -pickle.dump(classifier, open("model.pkl", "wb")) -with open('tokenizer.pk', 'wb') as fin: - pickle.dump(tokenizer, fin) -with open('frquencytransformer.pk', 'wb') as fin: - pickle.dump(frequency_transformer, fin) - - -test_counts = tokenizer.transform(['load movie data']) -test_frequencies = frequency_transformer.transform(test_counts) -ans = classifier.predict(test_frequencies) -print(ans) From ddff20fca5b4a427fc779a16cde0f694f872640a Mon Sep 17 00:00:00 2001 From: Maqbool ur Rahim Khan Date: Mon, 24 Jun 2019 01:17:08 +0200 Subject: [PATCH 04/12] Simple API added with NLE --- butler/main.py | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/butler/main.py b/butler/main.py index f05d003c3..17309203f 100644 --- a/butler/main.py +++ b/butler/main.py @@ -1,10 +1,43 @@ import fasttext +import flask +from flask import request + +app = flask.Flask(__name__) corpus = 'data.db' classifier = fasttext.supervised(corpus, 'model_chat', label_prefix='__label__') -msg = input('Enter text> ') -while msg != 'q': - print(classifier.predict([msg])) - msg = input('Enter text> ') +# Api to classify the user message. +@app.route('/predict', methods=['GET']) +def index(): + # Getting the user message from url parameter. + query = request.args.get('msg') + if query is None: + return flask.jsonify({ + 'error': 'No query parameter' + }) + + # creating a response object + # storing the model's prediction in the object + response = {} + + # Intent classification of message + cls = classifier.predict([query]) + cls = cls[0][0] + + # set the reply based on the class + if cls == 'greet': + response['message'] = 'Hello' + elif cls == 'help': + response['message'] = 'I can load datasets for you. \n I can visualise data too.' + elif cls == 'dataset': + response['message'] = 'Dataset loaded.' + elif cls == 'barchart': + response['message'] = 'Barchart drawn.' + + # returning the response object as json + return flask.jsonify(response) + + +app.run() \ No newline at end of file From 614b58765618743a2b4c46174cd950e0b04f3653 Mon Sep 17 00:00:00 2001 From: Maqbool ur Rahim Khan Date: Sun, 7 Jul 2019 14:55:39 +0200 Subject: [PATCH 05/12] context added --- butler/config_loader.py | 5 ++++ butler/intent_config.json | 12 +++++++++ butler/main.py | 54 ++++++++++++++++++++++++++++----------- 3 files changed, 56 insertions(+), 15 deletions(-) create mode 100644 butler/config_loader.py create mode 100644 butler/intent_config.json diff --git a/butler/config_loader.py b/butler/config_loader.py new file mode 100644 index 000000000..1ca0baec5 --- /dev/null +++ b/butler/config_loader.py @@ -0,0 +1,5 @@ +import json + +def config_loader(config_file_path): + config = open(config_file_path, 'r') + return json.loads(config.read()) \ No newline at end of file diff --git a/butler/intent_config.json b/butler/intent_config.json new file mode 100644 index 000000000..4c38313ae --- /dev/null +++ b/butler/intent_config.json @@ -0,0 +1,12 @@ +{ + "barchart": { + "parameters": [ + { + "x": "Ok then what is x-axis?" + }, + { + "y": "OK then what is y-axis?" + } + ] + } +} diff --git a/butler/main.py b/butler/main.py index 17309203f..f50c34cfd 100644 --- a/butler/main.py +++ b/butler/main.py @@ -1,19 +1,25 @@ import fasttext import flask from flask import request +from config_loader import config_loader app = flask.Flask(__name__) -corpus = 'data.db' +print('Loading intent configuration file.. ', end='') +intent_config = config_loader('intent_config.json') +print('Done') +print('Initializing classifier.. ', end='') +corpus = 'data.db' classifier = fasttext.supervised(corpus, 'model_chat', label_prefix='__label__') +print('Done') # Api to classify the user message. @app.route('/predict', methods=['GET']) def index(): # Getting the user message from url parameter. - query = request.args.get('msg') - if query is None: + msg = request.args.get('msg') + if msg is None: return flask.jsonify({ 'error': 'No query parameter' }) @@ -23,21 +29,39 @@ def index(): response = {} # Intent classification of message - cls = classifier.predict([query]) - cls = cls[0][0] - - # set the reply based on the class - if cls == 'greet': - response['message'] = 'Hello' - elif cls == 'help': - response['message'] = 'I can load datasets for you. \n I can visualise data too.' - elif cls == 'dataset': - response['message'] = 'Dataset loaded.' - elif cls == 'barchart': - response['message'] = 'Barchart drawn.' + cls = classifier.predict([msg])[0][0] + print(cls) + + if cls in intent_config: + if 'parameters' in intent_config.get(cls): + parameters = intent_config.get(cls).get('parameters') + + if 'param' not in request.args: + response['param'] = 0 + else: + response['param'] += 1 + + param = parameters[response.get('param')] + for k in param: + response['reply'] = param.get(k) + + response['complete'] = 0 + response['intent'] = cls + else: + response['complete'] = 1 + if cls == 'greet': + response['reply'] = 'Hello' + elif cls == 'help': + response['reply'] = 'I can load datasets for you. \n I can visualise data too.' + elif cls == 'dataset': + response['reply'] = 'Dataset loaded.' + elif cls == 'barchart': + response['reply'] = 'Barchart drawn.' + print('parameters missing') # returning the response object as json return flask.jsonify(response) +print('Starting server') app.run() \ No newline at end of file From f2524ed1baa2fc11e7d96dc41cf1d42465b3d4a5 Mon Sep 17 00:00:00 2001 From: Maqbool ur Rahim Khan Date: Sun, 7 Jul 2019 16:33:39 +0200 Subject: [PATCH 06/12] chatbot ui added --- butler/main.py | 8 ++++- butler/templates/index.html | 72 +++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 butler/templates/index.html diff --git a/butler/main.py b/butler/main.py index f50c34cfd..6e001ccfc 100644 --- a/butler/main.py +++ b/butler/main.py @@ -1,8 +1,9 @@ import fasttext import flask -from flask import request +from flask import request, render_template from config_loader import config_loader + app = flask.Flask(__name__) print('Loading intent configuration file.. ', end='') @@ -14,6 +15,11 @@ classifier = fasttext.supervised(corpus, 'model_chat', label_prefix='__label__') print('Done') + +@app.route('/') +def home(): + return render_template('index.html') + # Api to classify the user message. @app.route('/predict', methods=['GET']) def index(): diff --git a/butler/templates/index.html b/butler/templates/index.html new file mode 100644 index 000000000..6f3a1055f --- /dev/null +++ b/butler/templates/index.html @@ -0,0 +1,72 @@ + + + + + + + Chatbot - UI + + + +
+
+ +
+ + \ No newline at end of file From c2674247a18c766af17c7f77731d2920b16f23b2 Mon Sep 17 00:00:00 2001 From: Maqbool ur Rahim Khan Date: Sun, 7 Jul 2019 18:05:51 +0200 Subject: [PATCH 07/12] chatbot ui ajax reqeust added --- butler/templates/index.html | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/butler/templates/index.html b/butler/templates/index.html index 6f3a1055f..49c5ed8be 100644 --- a/butler/templates/index.html +++ b/butler/templates/index.html @@ -68,5 +68,26 @@
+ + \ No newline at end of file From 8fafc683588c11d147022a3727389b5558041c8f Mon Sep 17 00:00:00 2001 From: Maqbool ur Rahim Khan Date: Sun, 7 Jul 2019 22:27:08 +0200 Subject: [PATCH 08/12] reply now showing on ui --- butler/templates/index.html | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/butler/templates/index.html b/butler/templates/index.html index 49c5ed8be..a04b76bd0 100644 --- a/butler/templates/index.html +++ b/butler/templates/index.html @@ -10,7 +10,7 @@ body, textarea, input { - font-size: 13px; + font-size: 16px; } .chat { max-width: 300px; @@ -70,6 +70,12 @@