From 1ccd3f525675e777e8e7d9f395a6c8f8c411baf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B3zsef=20Sallai?= Date: Wed, 18 Aug 2021 00:04:20 +0300 Subject: [PATCH 1/5] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20migrate=20from=20rq=20?= =?UTF-8?q?to=20celery=20for=20async=20jobs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + Pipfile | 16 +- Pipfile.lock | 751 ++++++++++-------- README.md | 39 +- openbook/__init__.py | 3 + openbook/celery.py | 77 ++ openbook/settings.py | 50 +- openbook/urls.py | 1 - .../commands/worker_health_check.py | 2 + openbook_common/utils/rq_helpers.py | 2 + openbook_notifications/helpers.py | 3 +- .../{django_rq_jobs.py => jobs.py} | 5 +- openbook_posts/jobs.py | 17 +- openbook_posts/tests/views/test_post.py | 38 +- openbook_posts/tests/views/test_post_media.py | 59 +- openbook_posts/tests/views/test_posts.py | 13 +- requirements.txt | 1 + 17 files changed, 589 insertions(+), 489 deletions(-) create mode 100644 openbook/celery.py rename openbook_notifications/{django_rq_jobs.py => jobs.py} (93%) diff --git a/.gitignore b/.gitignore index 01492080..a80ba220 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ coverage.xml src/ static/ docker-compose.override.yml +*.db # Elastic Beanstalk Files .elasticbeanstalk/* diff --git a/Pipfile b/Pipfile index 2c94a5a7..70b640f5 100644 --- a/Pipfile +++ b/Pipfile @@ -8,38 +8,37 @@ polib = "*" pytz = "*" [packages] -django = "==2.2.16" +Django = "==2.2.16" python-dotenv = "*" djangorestframework = "*" sentry-sdk = "==0.10.2" mysqlclient = "*" -pillow = "*" +Pillow = "*" django-nose = "*" nose = "*" pinocchio = "*" coverage = "*" nose-exclude = "*" mixer = "*" -faker = "*" -"boto3" = "*" +Faker = "*" +boto3 = "*" django-storages = "==1.8" django-amazon-ses = "*" django-media-fixtures = "==0.0.3" django-modeltranslation = "*" bandit = "==1.5.1" -pyyaml = ">=4.2b1" +PyYAML = ">=4.2b1" python-magic = "*" safety = "*" -pyjwt = "*" +PyJWT = "*" django-imagekit = "*" rest-framework-generic-relations = "*" onesignal-sdk = "==1.1.0" -django-replicated = "*" +django_replicated = "*" django-cacheops = "*" django-redis = "*" django-extensions = "*" langdetect = "*" -django-rq = "*" redis = "*" django-rq-scheduler = "*" ffmpy = "*" @@ -56,6 +55,7 @@ halo = "*" watchdog = "*" spectra = "*" url-normalize = "*" +celery = "*" [pipenv] allow_prereleases = false diff --git a/Pipfile.lock b/Pipfile.lock index e82a8f26..f5546002 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "da4654a3ebb35168992cc3bb8fb89b9cb4e22de1f7d9f7e7c9eb8f1ecc0fbdf0" + "sha256": "761e5c228d2f131fb1a1bc84764e2fe8b1b2aaae5476e4e714cb42aa7d2b0b7e" }, "pipfile-spec": 6, "requires": {}, @@ -14,6 +14,13 @@ ] }, "default": { + "amqp": { + "hashes": [ + "sha256:03e16e94f2b34c31f8bf1206d8ddd3ccaa4c315f7f6a1879b7b1210d229568c2", + "sha256:493a2ac6788ce270a2f6a765b017299f60c1998f5a8617908ee9be082f7300fb" + ], + "version": "==5.0.6" + }, "appdirs": { "hashes": [ "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", @@ -38,49 +45,83 @@ "index": "pypi", "version": "==4.9.3" }, + "billiard": { + "hashes": [ + "sha256:299de5a8da28a783d51b197d496bef4f1595dd023a93a4f59dde1886ae905547", + "sha256:87103ea78fa6ab4d5c751c4909bcff74617d985de7fa8b672cf8618afd5a875b" + ], + "version": "==3.6.4.0" + }, "boto3": { "hashes": [ - "sha256:2e16f02c8b832d401d958d7ca0a14c5bc7da17827918e6b24e5bc43dce8f496e", - "sha256:ab5353a968a4e664b9da2dd950169b755066525fcbfdfc90e7e49c8333d95c19" + "sha256:6cc7011cb857fecee54884ff344d6b793cd22af51142f715706c757d26d02bb1", + "sha256:7405ae77ce4f2151fae1b542183f9c0f7ffb57c288b1f152819cfcb88e9cf297" ], "index": "pypi", - "version": "==1.16.0" + "version": "==1.18.22" }, "botocore": { "hashes": [ - "sha256:226effa72e3ddd0a802e812c0e204999393ca7982fee754cc0c770a7a1caef3a", - "sha256:9bf8586b69f20cf0a8ed1e27338cd10ce847751d1a2fd98b92662565c8a2df24" + "sha256:9c133caab58b04b4a9ab3f6523cc61cf815c1a5fde7b5ee279eefa48dc3a01d1", + "sha256:9df7a84840bcea10eb68f816d562c77656ec253a3a0dc3724e7e9ac086656e28" ], - "version": "==1.19.0" + "version": "==1.21.22" + }, + "celery": { + "hashes": [ + "sha256:8d9a3de9162965e97f8e8cc584c67aad83b3f7a267584fa47701ed11c3e0d4b0", + "sha256:9dab2170b4038f7bf10ef2861dbf486ddf1d20592290a1040f7b7a1259705d42" + ], + "index": "pypi", + "version": "==5.1.2" }, "certifi": { "hashes": [ - "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", - "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" + "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee", + "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8" ], - "version": "==2020.6.20" + "version": "==2021.5.30" }, - "chardet": { + "charset-normalizer": { "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b", + "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3" ], - "version": "==3.0.4" + "markers": "python_version >= '3'", + "version": "==2.0.4" }, "click": { "hashes": [ "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==7.1.2" }, + "click-didyoumean": { + "hashes": [ + "sha256:112229485c9704ff51362fe34b2d4f0b12fc71cc20f6d2b3afabed4b8bfa6aeb" + ], + "version": "==0.0.3" + }, + "click-plugins": { + "hashes": [ + "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b", + "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8" + ], + "version": "==1.1.1" + }, + "click-repl": { + "hashes": [ + "sha256:94b3fbbc9406a236f176e0506524b2937e4b23b6f4c0c0b2a0a83f8a64e9194b", + "sha256:cd12f68d745bf6151210790540b4cb064c7b13e571bc64b6957d98d120dacfd8" + ], + "version": "==0.2.0" + }, "colorama": { "hashes": [ "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==0.4.4" }, "colormath": { @@ -91,58 +132,68 @@ }, "coverage": { "hashes": [ - "sha256:0203acd33d2298e19b57451ebb0bed0ab0c602e5cf5a818591b4918b1f97d516", - "sha256:0f313707cdecd5cd3e217fc68c78a960b616604b559e9ea60cc16795c4304259", - "sha256:1c6703094c81fa55b816f5ae542c6ffc625fec769f22b053adb42ad712d086c9", - "sha256:1d44bb3a652fed01f1f2c10d5477956116e9b391320c94d36c6bf13b088a1097", - "sha256:280baa8ec489c4f542f8940f9c4c2181f0306a8ee1a54eceba071a449fb870a0", - "sha256:29a6272fec10623fcbe158fdf9abc7a5fa032048ac1d8631f14b50fbfc10d17f", - "sha256:2b31f46bf7b31e6aa690d4c7a3d51bb262438c6dcb0d528adde446531d0d3bb7", - "sha256:2d43af2be93ffbad25dd959899b5b809618a496926146ce98ee0b23683f8c51c", - "sha256:381ead10b9b9af5f64646cd27107fb27b614ee7040bb1226f9c07ba96625cbb5", - "sha256:47a11bdbd8ada9b7ee628596f9d97fbd3851bd9999d398e9436bd67376dbece7", - "sha256:4d6a42744139a7fa5b46a264874a781e8694bb32f1d76d8137b68138686f1729", - "sha256:50691e744714856f03a86df3e2bff847c2acede4c191f9a1da38f088df342978", - "sha256:530cc8aaf11cc2ac7430f3614b04645662ef20c348dce4167c22d99bec3480e9", - "sha256:582ddfbe712025448206a5bc45855d16c2e491c2dd102ee9a2841418ac1c629f", - "sha256:63808c30b41f3bbf65e29f7280bf793c79f54fb807057de7e5238ffc7cc4d7b9", - "sha256:71b69bd716698fa62cd97137d6f2fdf49f534decb23a2c6fc80813e8b7be6822", - "sha256:7858847f2d84bf6e64c7f66498e851c54de8ea06a6f96a32a1d192d846734418", - "sha256:78e93cc3571fd928a39c0b26767c986188a4118edc67bc0695bc7a284da22e82", - "sha256:7f43286f13d91a34fadf61ae252a51a130223c52bfefb50310d5b2deb062cf0f", - "sha256:86e9f8cd4b0cdd57b4ae71a9c186717daa4c5a99f3238a8723f416256e0b064d", - "sha256:8f264ba2701b8c9f815b272ad568d555ef98dfe1576802ab3149c3629a9f2221", - "sha256:9342dd70a1e151684727c9c91ea003b2fb33523bf19385d4554f7897ca0141d4", - "sha256:9361de40701666b034c59ad9e317bae95c973b9ff92513dd0eced11c6adf2e21", - "sha256:9669179786254a2e7e57f0ecf224e978471491d660aaca833f845b72a2df3709", - "sha256:aac1ba0a253e17889550ddb1b60a2063f7474155465577caa2a3b131224cfd54", - "sha256:aef72eae10b5e3116bac6957de1df4d75909fc76d1499a53fb6387434b6bcd8d", - "sha256:bd3166bb3b111e76a4f8e2980fa1addf2920a4ca9b2b8ca36a3bc3dedc618270", - "sha256:c1b78fb9700fc961f53386ad2fd86d87091e06ede5d118b8a50dea285a071c24", - "sha256:c3888a051226e676e383de03bf49eb633cd39fc829516e5334e69b8d81aae751", - "sha256:c5f17ad25d2c1286436761b462e22b5020d83316f8e8fcb5deb2b3151f8f1d3a", - "sha256:c851b35fc078389bc16b915a0a7c1d5923e12e2c5aeec58c52f4aa8085ac8237", - "sha256:cb7df71de0af56000115eafd000b867d1261f786b5eebd88a0ca6360cccfaca7", - "sha256:cedb2f9e1f990918ea061f28a0f0077a07702e3819602d3507e2ff98c8d20636", - "sha256:e8caf961e1b1a945db76f1b5fa9c91498d15f545ac0ababbe575cfab185d3bd8" - ], - "index": "pypi", - "version": "==5.3" + "sha256:004d1880bed2d97151facef49f08e255a20ceb6f9432df75f4eef018fdd5a78c", + "sha256:01d84219b5cdbfc8122223b39a954820929497a1cb1422824bb86b07b74594b6", + "sha256:040af6c32813fa3eae5305d53f18875bedd079960822ef8ec067a66dd8afcd45", + "sha256:06191eb60f8d8a5bc046f3799f8a07a2d7aefb9504b0209aff0b47298333302a", + "sha256:13034c4409db851670bc9acd836243aeee299949bd5673e11844befcb0149f03", + "sha256:13c4ee887eca0f4c5a247b75398d4114c37882658300e153113dafb1d76de529", + "sha256:184a47bbe0aa6400ed2d41d8e9ed868b8205046518c52464fde713ea06e3a74a", + "sha256:18ba8bbede96a2c3dde7b868de9dcbd55670690af0988713f0603f037848418a", + "sha256:1aa846f56c3d49205c952d8318e76ccc2ae23303351d9270ab220004c580cfe2", + "sha256:217658ec7187497e3f3ebd901afdca1af062b42cfe3e0dafea4cced3983739f6", + "sha256:24d4a7de75446be83244eabbff746d66b9240ae020ced65d060815fac3423759", + "sha256:2910f4d36a6a9b4214bb7038d537f015346f413a975d57ca6b43bf23d6563b53", + "sha256:2949cad1c5208b8298d5686d5a85b66aae46d73eec2c3e08c817dd3513e5848a", + "sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4", + "sha256:2cafbbb3af0733db200c9b5f798d18953b1a304d3f86a938367de1567f4b5bff", + "sha256:2e0d881ad471768bf6e6c2bf905d183543f10098e3b3640fc029509530091502", + "sha256:30c77c1dc9f253283e34c27935fded5015f7d1abe83bc7821680ac444eaf7793", + "sha256:3487286bc29a5aa4b93a072e9592f22254291ce96a9fbc5251f566b6b7343cdb", + "sha256:372da284cfd642d8e08ef606917846fa2ee350f64994bebfbd3afb0040436905", + "sha256:41179b8a845742d1eb60449bdb2992196e211341818565abded11cfa90efb821", + "sha256:44d654437b8ddd9eee7d1eaee28b7219bec228520ff809af170488fd2fed3e2b", + "sha256:4a7697d8cb0f27399b0e393c0b90f0f1e40c82023ea4d45d22bce7032a5d7b81", + "sha256:51cb9476a3987c8967ebab3f0fe144819781fca264f57f89760037a2ea191cb0", + "sha256:52596d3d0e8bdf3af43db3e9ba8dcdaac724ba7b5ca3f6358529d56f7a166f8b", + "sha256:53194af30d5bad77fcba80e23a1441c71abfb3e01192034f8246e0d8f99528f3", + "sha256:5fec2d43a2cc6965edc0bb9e83e1e4b557f76f843a77a2496cbe719583ce8184", + "sha256:6c90e11318f0d3c436a42409f2749ee1a115cd8b067d7f14c148f1ce5574d701", + "sha256:74d881fc777ebb11c63736622b60cb9e4aee5cace591ce274fb69e582a12a61a", + "sha256:7501140f755b725495941b43347ba8a2777407fc7f250d4f5a7d2a1050ba8e82", + "sha256:796c9c3c79747146ebd278dbe1e5c5c05dd6b10cc3bcb8389dfdf844f3ead638", + "sha256:869a64f53488f40fa5b5b9dcb9e9b2962a66a87dab37790f3fcfb5144b996ef5", + "sha256:8963a499849a1fc54b35b1c9f162f4108017b2e6db2c46c1bed93a72262ed083", + "sha256:8d0a0725ad7c1a0bcd8d1b437e191107d457e2ec1084b9f190630a4fb1af78e6", + "sha256:900fbf7759501bc7807fd6638c947d7a831fc9fdf742dc10f02956ff7220fa90", + "sha256:92b017ce34b68a7d67bd6d117e6d443a9bf63a2ecf8567bb3d8c6c7bc5014465", + "sha256:970284a88b99673ccb2e4e334cfb38a10aab7cd44f7457564d11898a74b62d0a", + "sha256:972c85d205b51e30e59525694670de6a8a89691186012535f9d7dbaa230e42c3", + "sha256:9a1ef3b66e38ef8618ce5fdc7bea3d9f45f3624e2a66295eea5e57966c85909e", + "sha256:af0e781009aaf59e25c5a678122391cb0f345ac0ec272c7961dc5455e1c40066", + "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf", + "sha256:b7895207b4c843c76a25ab8c1e866261bcfe27bfaa20c192de5190121770672b", + "sha256:c0891a6a97b09c1f3e073a890514d5012eb256845c451bd48f7968ef939bf4ae", + "sha256:c2723d347ab06e7ddad1a58b2a821218239249a9e4365eaff6649d31180c1669", + "sha256:d1f8bf7b90ba55699b3a5e44930e93ff0189aa27186e96071fac7dd0d06a1873", + "sha256:d1f9ce122f83b2305592c11d64f181b87153fc2c2bbd3bb4a3dde8303cfb1a6b", + "sha256:d314ed732c25d29775e84a960c3c60808b682c08d86602ec2c3008e1202e3bb6", + "sha256:d636598c8305e1f90b439dbf4f66437de4a5e3c31fdf47ad29542478c8508bbb", + "sha256:deee1077aae10d8fa88cb02c845cfba9b62c55e1183f52f6ae6a2df6a2187160", + "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c", + "sha256:f030f8873312a16414c0d8e1a1ddff2d3235655a2174e3648b4fa66b3f2f1079", + "sha256:f0b278ce10936db1a37e6954e15a3730bea96a0997c26d7fee88e6c396c2086d", + "sha256:f11642dddbb0253cc8853254301b51390ba0081750a8ac03f20ea8103f0c56b6" + ], + "index": "pypi", + "version": "==5.5" }, "croniter": { "hashes": [ - "sha256:4eeb32f173a714a78ab529f23a4d92b52e66c8d14b422202900230df74da82da", - "sha256:b9075573d9d18fdc4c67ad6741c4bfa4b446b1b1d7f03279757244c8a75abedf" + "sha256:0f97b361fe343301a8f66f852e7d84e4fb7f21379948f71e1bbfe10f5d015fbd", + "sha256:a70dfc9d52de9fc1a886128b9148c89dd9e76b67d55f46516ca94d2d73d58219" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.3.35" - }, - "decorator": { - "hashes": [ - "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760", - "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7" - ], - "version": "==4.4.2" + "version": "==1.0.15" }, "django": { "hashes": [ @@ -169,27 +220,27 @@ }, "django-cacheops": { "hashes": [ - "sha256:be48a8278abf124a4b3753d545f087bb67e221a5501ec758fc8cd1c4e915d9a0", - "sha256:f6a389fdfcdc98c735a775dde235a6192ed87e0309110129484c518974a328a4" + "sha256:78e161ebd96a32e28e19ec7da31f2afed9e62a79726b8b5f0ed12dd16c2e5841", + "sha256:ee38b969c9fc68f7c88e769b6c811e19563cca1ae08210d9f553ff758b6c3e17" ], "index": "pypi", - "version": "==5.0.1" + "version": "==6.0" }, "django-cursor-pagination": { "hashes": [ - "sha256:063fce867442c97798a247a306f9d889374eb630a3246b15493f08b72d3dbcd7", - "sha256:0a738fb24d1e52b28556054804a7cb6cc76c2c32019299d3321fc1c77be49c73" + "sha256:04ecc8bbfd702ff76f5987f2591e3b4308db85d4271084a2f731b181a4f36be7", + "sha256:7511cf668294ba61c7c9b289093aeec47a50fa3818e6feff50f5224ff200b664" ], "index": "pypi", - "version": "==0.1.4" + "version": "==0.1.5" }, "django-extensions": { "hashes": [ - "sha256:6809c89ca952f0e08d4e0766bc0101dfaf508d7649aced1180c091d737046ea7", - "sha256:dc663652ac9460fd06580a973576820430c6d428720e874ae46b041fa63e0efa" + "sha256:50de8977794a66a91575dd40f87d5053608f679561731845edbd325ceeb387e3", + "sha256:5f0fea7bf131ca303090352577a9e7f8bfbf5489bd9d9c8aea9401db28db34a0" ], "index": "pypi", - "version": "==3.0.9" + "version": "==3.1.3" }, "django-imagekit": { "hashes": [ @@ -208,17 +259,17 @@ }, "django-model-utils": { "hashes": [ - "sha256:9cf882e5b604421b62dbe57ad2b18464dc9c8f963fc3f9831badccae66c1139c", - "sha256:adf09e5be15122a7f4e372cb5a6dd512bbf8d78a23a90770ad0983ee9d909061" + "sha256:eb5dd05ef7d7ce6bc79cae54ea7c4a221f6f81e2aad7722933aee66489e7264b", + "sha256:ef7c440024e797796a3811432abdd2be8b5225ae64ef346f8bfc6de7d8e5d73c" ], - "version": "==4.0.0" + "version": "==4.1.1" }, "django-modeltranslation": { "hashes": [ - "sha256:93613fad41927e2849672cf75cf5bd7f6ed7c487b362e49a07ce3236c7ade474" + "sha256:fdbb32edcf92b82c934b336c5fd0e0ecf68bd9ce56fc55b172a617edf9c252cd" ], "index": "pypi", - "version": "==0.16" + "version": "==0.17.3" }, "django-nose": { "hashes": [ @@ -230,11 +281,11 @@ }, "django-ordered-model": { "hashes": [ - "sha256:29af6624cf3505daaf0df00e2df1d0726dd777b95e08f304d5ad0264092aa934", - "sha256:d867166ed4dd12501139e119cbbc5b4d19798a3e72740aef0af4879ba97102cf" + "sha256:3a8a0259bfd73a0c0b124932bb1fe59a6d2f4cbea096b20634d2a2d1f5d585cc", + "sha256:5aa58277b81b4ca93fb18caf15069af604bac5c5146d2c29aae56da07a86ef1b" ], "index": "pypi", - "version": "==3.4.1" + "version": "==3.4.3" }, "django-positions": { "hashes": [ @@ -253,11 +304,11 @@ }, "django-redis": { "hashes": [ - "sha256:1133b26b75baa3664164c3f44b9d5d133d1b8de45d94d79f38d1adc5b1d502e5", - "sha256:306589c7021e6468b2656edc89f62b8ba67e8d5a1c8877e2688042263daa7a63" + "sha256:048f665bbe27f8ff2edebae6aa9c534ab137f1e8fa7234147ef470df3f3aa9b8", + "sha256:97739ca9de3f964c51412d1d7d8aecdfd86737bb197fce6e1ff12620c63c97ee" ], "index": "pypi", - "version": "==4.12.1" + "version": "==5.0.0" }, "django-replicated": { "hashes": [ @@ -268,11 +319,10 @@ }, "django-rq": { "hashes": [ - "sha256:ee9aefba814ae00b6d6a566ed0cd2a0a090e0e42d90bb9c4f1c70d3cacb47a52", - "sha256:f0bd7ac3b8b4b4abb161646e80a8751c81664beaa810e92fdbcf9fb3e7d72727" + "sha256:23981f83c537178cbbf730f192c13e99cede2204e9d917b1c1c80c42215dd227", + "sha256:f09059ab37403a47c7933bca396fabb7f3058732d132462eade5333bc4bcac5f" ], - "index": "pypi", - "version": "==2.3.2" + "version": "==2.4.1" }, "django-rq-scheduler": { "hashes": [ @@ -292,34 +342,33 @@ }, "djangorestframework": { "hashes": [ - "sha256:5c5071fcbad6dce16f566d492015c829ddb0df42965d488b878594aabc3aed21", - "sha256:d54452aedebb4b650254ca092f9f4f5df947cb1de6ab245d817b08b4f4156249" + "sha256:6d1d59f623a5ad0509fe0d6bfe93cbdfe17b8116ebc8eda86d45f6e16e819aaf", + "sha256:f747949a8ddac876e879190df194b925c177cdeb725a099db1460872f7c0a7f2" ], "index": "pypi", - "version": "==3.12.1" + "version": "==3.12.4" }, "dparse": { "hashes": [ "sha256:a1b5f169102e1c894f9a7d5ccf6f9402a836a5d24be80a986c7ce9eaed78f367", "sha256:e953a25e44ebb60a5c6efc2add4420c177f1d8404509da88da9729202f306994" ], - "markers": "python_version >= '3.5'", "version": "==0.5.1" }, "faker": { "hashes": [ - "sha256:74b32991f8e08e4f2f84858b919eca253becfaec4b3fa5fcff7fdbd70d5d78b1", - "sha256:c2ce42dd8361e6d392276006d757532562463c8642b1086709584200b7fd7758" + "sha256:3e737576ff50cd98dfed643d6b3fd63194eca9df00e7f595960fe7da5220723d", + "sha256:b9e81e9da3dda3ac54189e034cfb943de576a259caeb226ccab43fcbcf6a7891" ], "index": "pypi", - "version": "==0.9.1" + "version": "==8.11.0" }, "ffmpy": { "hashes": [ - "sha256:c52a86f530f1caefddc15b50193f86d1bfe009855893b06e683d8a9445af99fb" + "sha256:757591581eee25b4a50ac9ffb9b58035a2794533db47e0512f53fb2d7b6f9adc" ], "index": "pypi", - "version": "==0.2.3" + "version": "==0.3.0" }, "filelock": { "hashes": [ @@ -330,66 +379,61 @@ }, "funcy": { "hashes": [ - "sha256:65b746fed572b392d886810a98d56939c6e0d545abb750527a717c21ced21008", - "sha256:c247c3d085e03a89974e0c9e8331e9a79db3768a263556ba896d6c92d665bba2" + "sha256:1d3fc5d42cf7564a6b2be04042d0df7a50c77903cf760a34786d0c9ebd659b25", + "sha256:2775409b7dc9106283f1224d97e6df5f2c02e7291c8caed72764f5a115dffb50" ], - "version": "==1.15" + "version": "==1.16" }, "gitdb": { "hashes": [ - "sha256:91f36bfb1ab7949b3b40e23736db18231bf7593edada2ba5c3a174a7b23657ac", - "sha256:c9e1f2d0db7ddb9a704c2a0217be31214e91a4fe1dea1efad19ae42ba0c285c9" + "sha256:6c4cc71933456991da20917998acbe6cf4fb41eeaab7d6d67fbc05ecd4c865b0", + "sha256:96bf5c08b157a666fec41129e6d327235284cca4c81e92109260f353ba138005" ], - "markers": "python_version >= '3.4'", - "version": "==4.0.5" + "version": "==4.0.7" }, "gitpython": { "hashes": [ - "sha256:138016d519bf4dd55b22c682c904ed2fd0235c3612b2f8f65ce218ff358deed8", - "sha256:a03f728b49ce9597a6655793207c6ab0da55519368ff5961e4a74ae475b9fa8e" + "sha256:b1e1c269deab1b08ce65403cf14e10d2ef1f6c89e33ea7c5e5bb0222ea593b8a", + "sha256:df0e072a200703a65387b0cfdf0466e3bab729c0458cf6b7349d0e9877636519" ], - "markers": "python_version >= '3.4'", - "version": "==3.1.9" + "version": "==3.1.20" }, "halo": { "hashes": [ - "sha256:78750d4778488b79b12104f3a966259579df6b42eec0d7736b3125bfc4c01b10", - "sha256:ad1fb4d7e29fe846d323bf4a8bf9e65fddcb24c10258c13e964cab13278f9568" + "sha256:5350488fb7d2aa7c31a1344120cee67a872901ce8858f60da7946cef96c208ab", + "sha256:7b67a3521ee91d53b7152d4ee3452811e1d2a6321975137762eb3d70063cc9d6" ], "index": "pypi", - "version": "==0.0.30" + "version": "==0.0.31" }, "idna": { "hashes": [ - "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", - "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.10" - }, - "importlib-metadata": { - "hashes": [ - "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da", - "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3" + "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a", + "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3" ], - "markers": "python_version < '3.8'", - "version": "==2.0.0" + "version": "==3.2" }, "jmespath": { "hashes": [ "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.0" }, + "kombu": { + "hashes": [ + "sha256:01481d99f4606f6939cdc9b637264ed353ee9e3e4f62cfb582324142c41a572d", + "sha256:e2dedd8a86c9077c350555153825a31e456a0dc20c15d5751f00137ec9c75f0a" + ], + "version": "==5.1.0" + }, "langdetect": { "hashes": [ - "sha256:363795ea005f1243c958e953245dac5d814fabdc025c9afa91588c5fa6b2fa83", - "sha256:f37495e63607865e47deed08d78f7f8e58172658216ff954b2f14671bcd87740" + "sha256:7cbc0746252f19e76f77c0b1690aadf01963be835ef0cd4b56dddf2a8f1dfc2a", + "sha256:cbc1fef89f8d062739774bd51eda3da3274006b3661d199c2655f6b3f6d605a0" ], "index": "pypi", - "version": "==1.0.8" + "version": "==1.0.9" }, "log-symbols": { "hashes": [ @@ -400,37 +444,29 @@ }, "mixer": { "hashes": [ - "sha256:3601e84134b79cd4ac663f79a39fc3005d6a3f37692e7062cc3c0a727ab79940", - "sha256:f1ccab542a1304d7b3da5d6ebe9f53de1aac2288427121a3a54a63e08c3a5367" + "sha256:6f842fdd10952355120c50217961da37d91a661fe7996ee117b43d050344416e", + "sha256:9acec419f11c9df286493910dbabc72c70d6c7c46648fa8c8c89d975af05e860" ], "index": "pypi", - "version": "==6.1.3" + "version": "==7.1.2" }, "mysqlclient": { "hashes": [ - "sha256:3f39855a4ad22805361e782cc4d1010ac74796225fa2d1c03cc16673ccdc983a", - "sha256:a6b5648f648b16335e3b1aaec93dc3fcc81a9a661180e306936437cc522c810b", - "sha256:edd42ccaa444b00702d5374b2f5f7585c9d0ce201917f15339f1c3cf91c1b1ed", - "sha256:fb2f75aea14722390d2d8ddf384ad99da708c707a96656210a7be8af20a2c5e5" + "sha256:0ac0dd759c4ca02c35a9fedc24bc982cf75171651e8187c2495ec957a87dfff7", + "sha256:3381ca1a4f37ff1155fcfde20836b46416d66531add8843f6aa6d968982731c3", + "sha256:71c4b330cf2313bbda0307fc858cc9055e64493ba9bf28454d25cf8b3ee8d7f5", + "sha256:f6ebea7c008f155baeefe16c56cd3ee6239f7a5a9ae42396c2f1860f08a7c432", + "sha256:fc575093cf81b6605bed84653e48b277318b880dc9becf42dd47fa11ffd3e2b6" ], "index": "pypi", - "version": "==2.0.1" - }, - "natsort": { - "hashes": [ - "sha256:a633464dc3a22b305df0f27abcb3e83515898aa1fd0ed2f9726c3571a27258cf", - "sha256:d3fd728a3ceb7c78a59aa8539692a75e37cbfd9b261d4d702e8016639820f90a" - ], - "markers": "python_version >= '3.4'", - "version": "==7.0.1" + "version": "==2.0.3" }, "networkx": { "hashes": [ - "sha256:7978955423fbc9639c10498878be59caf99b44dc304c2286162fd24b458c1602", - "sha256:8c5812e9f798d37c50570d15c4a69d5710a18d77bafc903ee9c5fba7454c616c" + "sha256:2306f1950ce772c5a59a57f5486d59bb9cab98497c45fc49cbc45ac0dec119bb", + "sha256:5fcb7004be69e8fbdf07dcb502efa5c77cadcaad6982164134eeb9721f826c2e" ], - "markers": "python_version >= '3.6'", - "version": "==2.5" + "version": "==2.6.2" }, "nose": { "hashes": [ @@ -450,35 +486,38 @@ }, "numpy": { "hashes": [ - "sha256:04c7d4ebc5ff93d9822075ddb1751ff392a4375e5885299445fcebf877f179d5", - "sha256:0bfd85053d1e9f60234f28f63d4a5147ada7f432943c113a11afcf3e65d9d4c8", - "sha256:0c66da1d202c52051625e55a249da35b31f65a81cb56e4c69af0dfb8fb0125bf", - "sha256:0d310730e1e793527065ad7dde736197b705d0e4c9999775f212b03c44a8484c", - "sha256:1669ec8e42f169ff715a904c9b2105b6640f3f2a4c4c2cb4920ae8b2785dac65", - "sha256:2117536e968abb7357d34d754e3733b0d7113d4c9f1d921f21a3d96dec5ff716", - "sha256:3733640466733441295b0d6d3dcbf8e1ffa7e897d4d82903169529fd3386919a", - "sha256:4339741994c775396e1a274dba3609c69ab0f16056c1077f18979bec2a2c2e6e", - "sha256:51ee93e1fac3fe08ef54ff1c7f329db64d8a9c5557e6c8e908be9497ac76374b", - "sha256:54045b198aebf41bf6bf4088012777c1d11703bf74461d70cd350c0af2182e45", - "sha256:58d66a6b3b55178a1f8a5fe98df26ace76260a70de694d99577ddeab7eaa9a9d", - "sha256:59f3d687faea7a4f7f93bd9665e5b102f32f3fa28514f15b126f099b7997203d", - "sha256:62139af94728d22350a571b7c82795b9d59be77fc162414ada6c8b6a10ef5d02", - "sha256:7118f0a9f2f617f921ec7d278d981244ba83c85eea197be7c5a4f84af80a9c3c", - "sha256:7c6646314291d8f5ea900a7ea9c4261f834b5b62159ba2abe3836f4fa6705526", - "sha256:967c92435f0b3ba37a4257c48b8715b76741410467e2bdb1097e8391fccfae15", - "sha256:9a3001248b9231ed73894c773142658bab914645261275f675d86c290c37f66d", - "sha256:aba1d5daf1144b956bc87ffb87966791f5e9f3e1f6fab3d7f581db1f5b598f7a", - "sha256:addaa551b298052c16885fc70408d3848d4e2e7352de4e7a1e13e691abc734c1", - "sha256:b594f76771bc7fc8a044c5ba303427ee67c17a09b36e1fa32bde82f5c419d17a", - "sha256:c35a01777f81e7333bcf276b605f39c872e28295441c265cd0c860f4b40148c1", - "sha256:cebd4f4e64cfe87f2039e4725781f6326a61f095bc77b3716502bed812b385a9", - "sha256:d526fa58ae4aead839161535d59ea9565863bb0b0bdb3cc63214613fb16aced4", - "sha256:d7ac33585e1f09e7345aa902c281bd777fdb792432d27fca857f39b70e5dd31c", - "sha256:e6ddbdc5113628f15de7e4911c02aed74a4ccff531842c583e5032f6e5a179bd", - "sha256:eb25c381d168daf351147713f49c626030dcff7a393d5caa62515d415a6071d8" - ], - "markers": "python_version >= '3.6'", - "version": "==1.19.2" + "sha256:09858463db6dd9f78b2a1a05c93f3b33d4f65975771e90d2cf7aadb7c2f66edf", + "sha256:209666ce9d4a817e8a4597cd475b71b4878a85fa4b8db41d79fdb4fdee01dde2", + "sha256:298156f4d3d46815eaf0fcf0a03f9625fc7631692bd1ad851517ab93c3168fc6", + "sha256:30fc68307c0155d2a75ad19844224be0f2c6f06572d958db4e2053f816b859ad", + "sha256:423216d8afc5923b15df86037c6053bf030d15cc9e3224206ef868c2d63dd6dc", + "sha256:426a00b68b0d21f2deb2ace3c6d677e611ad5a612d2c76494e24a562a930c254", + "sha256:466e682264b14982012887e90346d33435c984b7fead7b85e634903795c8fdb0", + "sha256:51a7b9db0a2941434cd930dacaafe0fc9da8f3d6157f9d12f761bbde93f46218", + "sha256:52a664323273c08f3b473548bf87c8145b7513afd63e4ebba8496ecd3853df13", + "sha256:550564024dc5ceee9421a86fc0fb378aa9d222d4d0f858f6669eff7410c89bef", + "sha256:5de64950137f3a50b76ce93556db392e8f1f954c2d8207f78a92d1f79aa9f737", + "sha256:640c1ccfd56724f2955c237b6ccce2e5b8607c3bc1cc51d3933b8c48d1da3723", + "sha256:7fdc7689daf3b845934d67cb221ba8d250fdca20ac0334fea32f7091b93f00d3", + "sha256:805459ad8baaf815883d0d6f86e45b3b0b67d823a8f3fa39b1ed9c45eaf5edf1", + "sha256:92a0ab128b07799dd5b9077a9af075a63467d03ebac6f8a93e6440abfea4120d", + "sha256:9f2dc79c093f6c5113718d3d90c283f11463d77daa4e83aeeac088ec6a0bda52", + "sha256:a5109345f5ce7ddb3840f5970de71c34a0ff7fceb133c9441283bb8250f532a3", + "sha256:a55e4d81c4260386f71d22294795c87609164e22b28ba0d435850fbdf82fc0c5", + "sha256:a9da45b748caad72ea4a4ed57e9cd382089f33c5ec330a804eb420a496fa760f", + "sha256:b160b9a99ecc6559d9e6d461b95c8eec21461b332f80267ad2c10394b9503496", + "sha256:b342064e647d099ca765f19672696ad50c953cac95b566af1492fd142283580f", + "sha256:b5e8590b9245803c849e09bae070a8e1ff444f45e3f0bed558dd722119eea724", + "sha256:bf75d5825ef47aa51d669b03ce635ecb84d69311e05eccea083f31c7570c9931", + "sha256:c01b59b33c7c3ba90744f2c695be571a3bd40ab2ba7f3d169ffa6db3cfba614f", + "sha256:d96a6a7d74af56feb11e9a443150216578ea07b7450f7c05df40eec90af7f4a7", + "sha256:dd0e3651d210068d13e18503d75aaa45656eef51ef0b261f891788589db2cc38", + "sha256:e167b9805de54367dcb2043519382be541117503ce99e3291cc9b41ca0a83557", + "sha256:e42029e184008a5fd3d819323345e25e2337b0ac7f5c135b7623308530209d57", + "sha256:f545c082eeb09ae678dd451a1b1dbf17babd8a0d7adea02897a76e639afca310", + "sha256:fde50062d67d805bc96f1a9ecc0d37bfc2a8f02b937d2c50824d186aa91f2419" + ], + "version": "==1.21.2" }, "onesignal-sdk": { "hashes": [ @@ -490,25 +529,17 @@ }, "packaging": { "hashes": [ - "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", - "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" + "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7", + "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==20.4" - }, - "pathtools": { - "hashes": [ - "sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0" - ], - "version": "==0.1.2" + "version": "==21.0" }, "pbr": { "hashes": [ - "sha256:5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9", - "sha256:b236cde0ac9a6aedd5e3c34517b423cd4fd97ef723849da6b0d2231142d89c00" + "sha256:42df03e7797b796625b1029c0400279c7c34fd7df24a7d7818a1abb5b38710dd", + "sha256:c68c661ac5cc81058ac94247278eeda6d2e6aecb3e227b0387c30d277e7ef8d4" ], - "markers": "python_version >= '2.6'", - "version": "==5.5.1" + "version": "==5.6.0" }, "pilkit": { "hashes": [ @@ -518,110 +549,143 @@ }, "pillow": { "hashes": [ - "sha256:04d984e45a0b9815f4b407e8aadb50f25fbb82a605d89db927376e94c3adf371", - "sha256:06e730451b70471c08b8a0ee7f18e7e1df310dba9c780bbfb730a13102b143db", - "sha256:1f59596af2b3d64a9e43f9d6509b7a51db744d0eecc23297617c604e6823c6ae", - "sha256:233513465a2f25fce537b965621866da3d1f02e15708f371dd4e19f0fb7b7711", - "sha256:2696f1a6402c1a42ed12c5cd8adfb4b381c32d41e35a34b8ee544309ef854172", - "sha256:2ca55a4443b463eec90528ac27be14d226b1c2b972178bc7d4d282ce89e47b6a", - "sha256:30615e9115f976e00a938a28c7152562e8cf8e221ddacf4446dd8b20c0d97333", - "sha256:3a77e7b9f8991b81d7be8e0b2deab05013cf3ebb24ac2b863d2979acb68c73dd", - "sha256:54667c8ab16658cc0b7d824d8706b440d4db8382a3561042758bdfd48ca99298", - "sha256:59304c67d12394815331eda95ec892bf54ad95e0aa7bc1ccd8e0a4a5a25d4bf3", - "sha256:594f2f25b7bcfd9542c41b9df156fb5104f19f5fcefa51b1447f1d9f64c9cc14", - "sha256:5b5dde5dcedc4e6f5a71d7654a3c6e189ced82e97d7896b1ca5a5c5e4e0e916f", - "sha256:6bcea85f93fb2c94a1bcd35704c348a929a7fb24a0ec0cc2b9fcbb0046b87176", - "sha256:718d7f0eb3351052023b33fe0f83fc9e3beeb7cbacbd0ff2b52524e2153e4598", - "sha256:7c4a7ee37027ca716f42726b6f9fc491c13c843c7af559e0767dfab1ae9682d4", - "sha256:87a855b64a9b692604f6339baa4f9913d06838df1b4ccf0cb899dd18f56ec03c", - "sha256:8c006d52365c0a6bb41a07f9c8f9f458ae8170e0af3b8c49bf7089347066b97b", - "sha256:8e29701229705615d3dcfc439c7c46f40f913e57c7fe322b1efc30d3f37d1287", - "sha256:9b5b41737853bc49943864d5980dfb401a09e78ddb471e71291810ccdeadd712", - "sha256:b04569ff215b85ce3e2954979d2d5e0bf84007e43ddcf84b632fc6bc18e07909", - "sha256:b731d45764349313bd956c07bdc1d43803bb0ad2b11354328a074e416c7d84bc", - "sha256:c12e33cb17e2e12049a49b77696ee479791a4e44e541fdc393ae043e1246389f", - "sha256:c41442c3814afeba1f6f16fd70cdf312a2c73c6dee8dc3ac8926bb115713ad1d", - "sha256:c4d743c5c91424965707c9c8edc58b7cb43c127dcaf191fbcd304e2082eef56a", - "sha256:d6766fd28f4f47cf93280a57e3dc6a9d11bdada1a6e9f019b8c62b12bbc86f6a", - "sha256:d904570afcdbec40eb6bdbe24cba8d95c0215a2c0cbbc9c16301045bc8504c1f", - "sha256:e674be2f349ea810e221b0113bd4491f53584ac848d5bcc3b62443cfa11d9c40", - "sha256:e6ac40f1a62a227eb00226eb64c9c82bc878a3ed700b5414d34c9be57be87e87", - "sha256:f5270369c799b4405ed47d45c88c09fbd7942fc9fb9891c0dabf0b8c751b625d" - ], - "index": "pypi", - "version": "==8.0.0" + "sha256:0b2efa07f69dc395d95bb9ef3299f4ca29bcb2157dc615bae0b42c3c20668ffc", + "sha256:114f816e4f73f9ec06997b2fde81a92cbf0777c9e8f462005550eed6bae57e63", + "sha256:147bd9e71fb9dcf08357b4d530b5167941e222a6fd21f869c7911bac40b9994d", + "sha256:15a2808e269a1cf2131930183dcc0419bc77bb73eb54285dde2706ac9939fa8e", + "sha256:196560dba4da7a72c5e7085fccc5938ab4075fd37fe8b5468869724109812edd", + "sha256:1c03e24be975e2afe70dfc5da6f187eea0b49a68bb2b69db0f30a61b7031cee4", + "sha256:1fd5066cd343b5db88c048d971994e56b296868766e461b82fa4e22498f34d77", + "sha256:29c9569049d04aaacd690573a0398dbd8e0bf0255684fee512b413c2142ab723", + "sha256:2b6dfa068a8b6137da34a4936f5a816aba0ecc967af2feeb32c4393ddd671cba", + "sha256:2cac53839bfc5cece8fdbe7f084d5e3ee61e1303cccc86511d351adcb9e2c792", + "sha256:2ee77c14a0299d0541d26f3d8500bb57e081233e3fa915fa35abd02c51fa7fae", + "sha256:37730f6e68bdc6a3f02d2079c34c532330d206429f3cee651aab6b66839a9f0e", + "sha256:3f08bd8d785204149b5b33e3b5f0ebbfe2190ea58d1a051c578e29e39bfd2367", + "sha256:479ab11cbd69612acefa8286481f65c5dece2002ffaa4f9db62682379ca3bb77", + "sha256:4bc3c7ef940eeb200ca65bd83005eb3aae8083d47e8fcbf5f0943baa50726856", + "sha256:660a87085925c61a0dcc80efb967512ac34dbb256ff7dd2b9b4ee8dbdab58cf4", + "sha256:67b3666b544b953a2777cb3f5a922e991be73ab32635666ee72e05876b8a92de", + "sha256:70af7d222df0ff81a2da601fab42decb009dc721545ed78549cb96e3a1c5f0c8", + "sha256:75e09042a3b39e0ea61ce37e941221313d51a9c26b8e54e12b3ececccb71718a", + "sha256:8960a8a9f4598974e4c2aeb1bff9bdd5db03ee65fd1fce8adf3223721aa2a636", + "sha256:9364c81b252d8348e9cc0cb63e856b8f7c1b340caba6ee7a7a65c968312f7dab", + "sha256:969cc558cca859cadf24f890fc009e1bce7d7d0386ba7c0478641a60199adf79", + "sha256:9a211b663cf2314edbdb4cf897beeb5c9ee3810d1d53f0e423f06d6ebbf9cd5d", + "sha256:a17ca41f45cf78c2216ebfab03add7cc350c305c38ff34ef4eef66b7d76c5229", + "sha256:a2f381932dca2cf775811a008aa3027671ace723b7a38838045b1aee8669fdcf", + "sha256:a4eef1ff2d62676deabf076f963eda4da34b51bc0517c70239fafed1d5b51500", + "sha256:c088a000dfdd88c184cc7271bfac8c5b82d9efa8637cd2b68183771e3cf56f04", + "sha256:c0e0550a404c69aab1e04ae89cca3e2a042b56ab043f7f729d984bf73ed2a093", + "sha256:c11003197f908878164f0e6da15fce22373ac3fc320cda8c9d16e6bba105b844", + "sha256:c2a5ff58751670292b406b9f06e07ed1446a4b13ffced6b6cab75b857485cbc8", + "sha256:c35d09db702f4185ba22bb33ef1751ad49c266534339a5cebeb5159d364f6f82", + "sha256:c379425c2707078dfb6bfad2430728831d399dc95a7deeb92015eb4c92345eaf", + "sha256:cc866706d56bd3a7dbf8bac8660c6f6462f2f2b8a49add2ba617bc0c54473d83", + "sha256:d0da39795049a9afcaadec532e7b669b5ebbb2a9134576ebcc15dd5bdae33cc0", + "sha256:f156d6ecfc747ee111c167f8faf5f4953761b5e66e91a4e6767e548d0f80129c", + "sha256:f4ebde71785f8bceb39dcd1e7f06bcc5d5c3cf48b9f69ab52636309387b097c8", + "sha256:fc214a6b75d2e0ea7745488da7da3c381f41790812988c7a92345978414fad37", + "sha256:fd7eef578f5b2200d066db1b50c4aa66410786201669fb76d5238b007918fb24", + "sha256:ff04c373477723430dce2e9d024c708a047d44cf17166bf16e604b379bf0ca14" + ], + "index": "pypi", + "version": "==8.3.1" }, "pinocchio": { "hashes": [ - "sha256:1c73db81274d5c8cb50de768fb5735c6c6702891094acb4b08c0920b3758befd", - "sha256:bc53568703bc8e22d0b96010be657a5ebc6ca445defa45878568a0aef992c343" + "sha256:c09b190ed4e728e4782a17de17200780fac91b7cf9b604fd1892cbf5a453b81d", + "sha256:fc512a888f828987521a9beec20d5416746fb5209bc74de410153300dc2b2cd6" ], "index": "pypi", - "version": "==0.4.2" + "version": "==0.4.3" + }, + "prompt-toolkit": { + "hashes": [ + "sha256:08360ee3a3148bdb5163621709ee322ec34fc4375099afa4bbf751e9b7b7fa4f", + "sha256:7089d8d2938043508aa9420ec18ce0922885304cddae87fb96eebca942299f88" + ], + "version": "==3.0.19" }, "pyjwt": { "hashes": [ - "sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e", - "sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96" + "sha256:934d73fbba91b0483d3857d1aff50e96b2a892384ee2c17417ed3203f173fca1", + "sha256:fba44e7898bbca160a2b2b501f492824fc8382485d3a6f11ba5d0c1937ce6130" ], "index": "pypi", - "version": "==1.7.1" + "version": "==2.1.0" }, "pyparsing": { "hashes": [ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.4.7" }, "python-dateutil": { "hashes": [ - "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", - "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.8.1" + "version": "==2.8.2" }, "python-dotenv": { "hashes": [ - "sha256:8c10c99a1b25d9a68058a1ad6f90381a62ba68230ca93966882a4dbc3bc9c33d", - "sha256:c10863aee750ad720f4f43436565e4c1698798d763b63234fb5021b6c616e423" + "sha256:aae25dc1ebe97c420f50b81fb0e5c949659af713f31fdb63c749ca68748f34b1", + "sha256:f521bc2ac9a8e03c736f62911605c5d83970021e3fa95b37d769e2bbbe9b6172" ], "index": "pypi", - "version": "==0.14.0" + "version": "==0.19.0" }, "python-magic": { "hashes": [ - "sha256:356efa93c8899047d1eb7d3eb91e871ba2f5b1376edbaf4cc305e3c872207355", - "sha256:b757db2a5289ea3f1ced9e60f072965243ea43a2221430048fd8cacab17be0ce" + "sha256:4fec8ee805fea30c07afccd1592c0f17977089895bdfaae5fec870a84e997626", + "sha256:de800df9fb50f8ec5974761054a708af6e4246b03b4bdaee993f948947b0ebcf" ], "index": "pypi", - "version": "==0.4.18" + "version": "==0.4.24" }, "pytz": { "hashes": [ - "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed", - "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048" + "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da", + "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798" ], - "version": "==2020.1" + "version": "==2021.1" }, "pyyaml": { "hashes": [ - "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", - "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", - "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", - "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", - "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", - "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", - "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", - "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", - "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", - "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", - "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" - ], - "index": "pypi", - "version": "==5.3.1" + "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf", + "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696", + "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393", + "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77", + "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922", + "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5", + "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8", + "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10", + "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc", + "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018", + "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e", + "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253", + "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347", + "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183", + "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541", + "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb", + "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185", + "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc", + "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db", + "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa", + "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46", + "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122", + "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b", + "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63", + "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df", + "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc", + "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247", + "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6", + "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0" + ], + "index": "pypi", + "version": "==5.4.1" }, "redis": { "hashes": [ @@ -633,11 +697,10 @@ }, "requests": { "hashes": [ - "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", - "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" + "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24", + "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==2.24.0" + "version": "==2.26.0" }, "requests-file": { "hashes": [ @@ -656,33 +719,32 @@ }, "rq": { "hashes": [ - "sha256:6e32a39d467ffc56fc18f0f0f10abd6aa258895dbac03af31e38fe0c2337aab8", - "sha256:fc23788eedc39cad3c10630af1694a550eba2f8519e57e396fd91b1dba0a7d99" + "sha256:7af1e9706dbe6f1eac16dffacd8271ec27c1369950941f14dab6bb08a62979d7", + "sha256:bdfef943de838955e474cfd0e25b9b8c53ed4b9c361fe4bb11cf56d17a87acc5" ], - "markers": "python_version >= '3.5'", - "version": "==1.5.2" + "version": "==1.9.0" }, "rq-scheduler": { "hashes": [ - "sha256:de2e2754a05549dadbe7422d2d1fa5ad0c1d556280515af9c3b9c1eba0db5ddb", - "sha256:f19486671473eba42095163f2ae829d5dcb046dde01628233871ee830c1c3c2e" + "sha256:da94e9b6badf112995ff38fe16192e4f4c43c412b3c9614684ed8c8f7ca517d2", + "sha256:db79bb56cdbc4f7ffdd8bd659e389e91aa0db9c1abf002dc46f5dd6f0dbd2910" ], - "version": "==0.10.0" + "version": "==0.11.0" }, "s3transfer": { "hashes": [ - "sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13", - "sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db" + "sha256:50ed823e1dc5868ad40c8dc92072f757aa0e653a192845c94a3b676f4a62da4c", + "sha256:9c1dc369814391a6bda20ebbf4b70a0f34630592c9aa520856bf384916af2803" ], - "version": "==0.3.3" + "version": "==0.5.0" }, "safety": { "hashes": [ - "sha256:23bf20690d4400edc795836b0c983c2b4cbbb922233108ff925b7dd7750f00c9", - "sha256:86c1c4a031fe35bd624fce143fbe642a0234d29f7cbf7a9aa269f244a955b087" + "sha256:30e394d02a20ac49b7f65292d19d38fa927a8f9582cdfd3ad1adbbc66c641ad5", + "sha256:5f802ad5df5614f9622d8d71fedec2757099705c2356f862847c58c6dfe13e84" ], "index": "pypi", - "version": "==1.9.0" + "version": "==1.10.3" }, "sentry-sdk": { "hashes": [ @@ -701,27 +763,25 @@ }, "six": { "hashes": [ - "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", - "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.15.0" + "version": "==1.16.0" }, "smmap": { "hashes": [ - "sha256:54c44c197c819d5ef1991799a7e30b662d1e520f2ac75c9efbeb54a742214cf4", - "sha256:9c98bbd1f9786d22f14b3d4126894d56befb835ec90cef151af566c7e19b5d24" + "sha256:7e65386bd122d45405ddf795637b7f7d2b532e7e401d46bbe3fb49b9986d5182", + "sha256:a9a7479e4c572e2e775c404dcd3080c8dc49f39918c2cf74913d30c4c478e3c2" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==3.0.4" + "version": "==4.0.0" }, "soupsieve": { "hashes": [ - "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", - "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232" + "sha256:052774848f448cf19c7e959adf5566904d525f33a3f8b6ba6f6f8f26ec7de0cc", + "sha256:c2c1c2d44f158cdbddab7824a9af8c4f83c76b1e23e049479aa432feb6c4c23b" ], "markers": "python_version >= '3.0'", - "version": "==2.0.1" + "version": "==2.2.1" }, "spectra": { "hashes": [ @@ -742,16 +802,14 @@ "sha256:017cde379adbd6a1f15a61873f43e8274179378e95ef3fede90b5aa64d304ed0", "sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8" ], - "markers": "python_version >= '3.5'", "version": "==0.4.1" }, "stevedore": { "hashes": [ - "sha256:5e1ab03eaae06ef6ce23859402de785f08d97780ed774948ef16c4652c41bc62", - "sha256:f845868b3a3a77a2489d226568abe7328b5c2d4f6a011cc759dfa99144a521f0" + "sha256:3a5bbd0652bf552748871eaa73a4a8dc2899786bc497a2aa1fcb4dcdb0debeee", + "sha256:50d7b78fbaf0d04cd62411188fa7eedcb03eb7f4c4b37005615ceebe582aa82a" ], - "markers": "python_version >= '3.6'", - "version": "==3.2.2" + "version": "==3.3.0" }, "termcolor": { "hashes": [ @@ -761,64 +819,105 @@ }, "text-unidecode": { "hashes": [ - "sha256:5a1375bb2ba7968740508ae38d92e1f889a0832913cb1c447d5e2046061a396d", - "sha256:801e38bd550b943563660a91de8d4b6fa5df60a542be9093f7abf819f86050cc" + "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", + "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93" ], - "version": "==1.2" + "version": "==1.3" }, "tldextract": { "hashes": [ - "sha256:ab0e38977a129c72729476d5f8c85a8e1f8e49e9202e1db8dca76e95da7be9a8", - "sha256:c2a8a392edf3ea6fa8be80930f04c3ac29e91fa604cb2139bdf6a37fc1e1ac6d" + "sha256:cfae9bc8bda37c3e8c7c8639711ad20e95dc85b207a256b60b0b23d7ff5540ea", + "sha256:e57f22b6d00a28c21673d2048112f1bdcb6a14d4711568305f6bb96cf5bb53a1" ], "index": "pypi", - "version": "==2.2.3" + "version": "==3.1.0" }, "toml": { "hashes": [ - "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", - "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88" + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "version": "==0.10.2" + }, + "typing-extensions": { + "hashes": [ + "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497", + "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342", + "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" ], - "version": "==0.10.1" + "markers": "python_version < '3.10'", + "version": "==3.10.0.0" }, "uritools": { "hashes": [ - "sha256:405917a31ce58a57c8ccd0e4ea290f38baf2f4823819c3688f5331f1aee4ccb0", - "sha256:5ba20a5de979a6a65b3d55998bee86ca1d97075d0f22923b85590b5e39c2b307" + "sha256:28ffef82ce3b2793237d36e45aa7cde28dae6502f6a93fdbd05ede401520e279", + "sha256:576737664f51f82d5c2a98e25f6c5da73a57cc88326dbb686fd6c5d06ebd6c29" ], - "markers": "python_version ~= '3.5'", - "version": "==3.0.0" + "version": "==3.0.2" }, "url-normalize": { "hashes": [ - "sha256:1709cb4739e496f9f807a894e361915792f273538e250b1ab7da790544a665c3", - "sha256:1bd7085349dcdf06e52194d0f75ff99fff2eeed0da85a50e4cc2346452c1b8bc" + "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2", + "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed" ], "index": "pypi", - "version": "==1.4.2" + "version": "==1.4.3" }, "urlextract": { "hashes": [ - "sha256:5ad1c933bf9f066976a9f8d24f308aef6fe79688d0e4158cb516bd33576d42a8", - "sha256:96b501544f6be1242903428573615d7ec9d79d640576a2b3a2d202d2ad782499" + "sha256:483f4dadbc749be7fd3a3305ec6c89d5682de1be739e0ef299148a1e4c62ea94", + "sha256:cb13ae8acc053899c0bf1c134fef99864f276562b67f878fb10a54608ec4612e" ], "index": "pypi", - "version": "==1.1.0" + "version": "==1.3.0" }, "urllib3": { "hashes": [ - "sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2", - "sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e" + "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4", + "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f" ], - "markers": "python_version != '3.4'", - "version": "==1.25.11" + "version": "==1.26.6" }, - "watchdog": { + "vine": { "hashes": [ - "sha256:4214e1379d128b0588021880ccaf40317ee156d4603ac388b9adcf29165e0c04" + "sha256:4c9dceab6f76ed92105027c49c823800dd33cacce13bdedc5b914e3514b7fb30", + "sha256:7d3b1624a953da82ef63462013bbd271d3eb75751489f9807598e8f340bd637e" ], - "index": "pypi", - "version": "==0.10.3" + "version": "==5.0.0" + }, + "watchdog": { + "hashes": [ + "sha256:0bcdf7b99b56a3ae069866c33d247c9994ffde91b620eaf0306b27e099bd1ae0", + "sha256:0bcfe904c7d404eb6905f7106c54873503b442e8e918cc226e1828f498bdc0ca", + "sha256:201cadf0b8c11922f54ec97482f95b2aafca429c4c3a4bb869a14f3c20c32686", + "sha256:3a7d242a7963174684206093846537220ee37ba9986b824a326a8bb4ef329a33", + "sha256:3e305ea2757f81d8ebd8559d1a944ed83e3ab1bdf68bcf16ec851b97c08dc035", + "sha256:431a3ea70b20962e6dee65f0eeecd768cd3085ea613ccb9b53c8969de9f6ebd2", + "sha256:44acad6f642996a2b50bb9ce4fb3730dde08f23e79e20cd3d8e2a2076b730381", + "sha256:54e057727dd18bd01a3060dbf5104eb5a495ca26316487e0f32a394fd5fe725a", + "sha256:6fe9c8533e955c6589cfea6f3f0a1a95fb16867a211125236c82e1815932b5d7", + "sha256:85b851237cf3533fabbc034ffcd84d0fa52014b3121454e5f8b86974b531560c", + "sha256:8805a5f468862daf1e4f4447b0ccf3acaff626eaa57fbb46d7960d1cf09f2e6d", + "sha256:9628f3f85375a17614a2ab5eac7665f7f7be8b6b0a2a228e6f6a2e91dd4bfe26", + "sha256:a12539ecf2478a94e4ba4d13476bb2c7a2e0a2080af2bb37df84d88b1b01358a", + "sha256:acc4e2d5be6f140f02ee8590e51c002829e2c33ee199036fcd61311d558d89f4", + "sha256:b5fc5c127bad6983eecf1ad117ab3418949f18af9c8758bd10158be3647298a9", + "sha256:b8ddb2c9f92e0c686ea77341dcb58216fa5ff7d5f992c7278ee8a392a06e86bb", + "sha256:bf84bd94cbaad8f6b9cbaeef43080920f4cb0e61ad90af7106b3de402f5fe127", + "sha256:d9456f0433845e7153b102fffeb767bde2406b76042f2216838af3b21707894e", + "sha256:e4929ac2aaa2e4f1a30a36751160be391911da463a8799460340901517298b13", + "sha256:e5236a8e8602ab6db4b873664c2d356c365ab3cac96fbdec4970ad616415dd45", + "sha256:fd8c595d5a93abd441ee7c5bb3ff0d7170e79031520d113d6f401d0cf49d7c8f" + ], + "index": "pypi", + "version": "==2.1.3" + }, + "wcwidth": { + "hashes": [ + "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784", + "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83" + ], + "version": "==0.2.5" }, "webpreview": { "hashes": [ @@ -826,31 +925,23 @@ ], "index": "pypi", "version": "==1.6.0" - }, - "zipp": { - "hashes": [ - "sha256:16522f69653f0d67be90e8baa4a46d66389145b734345d68a257da53df670903", - "sha256:c1532a8030c32fd52ff6a288d855fe7adef5823ba1d26a29a68fd6314aa72baa" - ], - "markers": "python_version >= '3.6'", - "version": "==3.3.1" } }, "develop": { "polib": { "hashes": [ - "sha256:93b730477c16380c9a96726c54016822ff81acfa553977fdd131f2b90ba858d7", - "sha256:fad87d13696127ffb27ea0882d6182f1a9cf8a5e2b37a587751166c51e5a332a" + "sha256:d3ee85e0c6788f789353416b1612c6c92d75fe6ccfac0029711974d6abd0f86d", + "sha256:e02c355ae5e054912e3b0d16febc56510eff7e49d60bf22aecb463bd2f2a2dfa" ], "index": "pypi", - "version": "==1.1.0" + "version": "==1.1.1" }, "pytz": { "hashes": [ - "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed", - "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048" + "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da", + "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798" ], - "version": "==2020.1" + "version": "==2021.1" } } } diff --git a/README.md b/README.md index 877e320f..86112dbf 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ The API server for Okuna. ## Project overview -The project is a [Django](https://www.djangoproject.com/start/) application. +The project is a [Django](https://www.djangoproject.com/start/) application. ## Contributing @@ -92,7 +92,7 @@ Please read and follow our [Code of Conduct](https://github.com/OkunaOrg/okuna-a #### License -Every contribution accepted is licensed under [AGPL v3.0](http://www.gnu.org/licenses/agpl-3.0.html) or any later version. +Every contribution accepted is licensed under [AGPL v3.0](http://www.gnu.org/licenses/agpl-3.0.html) or any later version. You must be careful to not include any code that can not be licensed under this license. Please read carefully [our license](https://github.com/OkunaOrg/okuna-api/blob/master/LICENSE.txt) and ask us if you have any questions. @@ -134,13 +134,13 @@ You can use the CLI in two modes. 1. [Full mode](#full-mode) - **Best for Okuna mobile/web app development** 2. [Services-only mode](#services-only-mode) - **Best for Okuna API development** -Depending on the kind of development you would like to do, follow the instructions below for your chosen mode. +Depending on the kind of development you would like to do, follow the instructions below for your chosen mode. ### Full mode **Best for Mobile/web app development** -This mode brings a whole Okuna instance up, ready to use with a local Okuna mobile/web app. +This mode brings a whole Okuna instance up, ready to use with a local Okuna mobile/web app. #### Installation @@ -179,7 +179,7 @@ If the process was abruptly terminated and Okuna is still running in the backgro ```bash python okuna-cli.py down-full -``` +``` ### Services-only mode @@ -187,7 +187,7 @@ python okuna-cli.py down-full The Okuna services are a SQL server, a Redis server, a job scheduler server and a job worker server. -This mode brings these services up but **not** the Okuna API itself, +This mode brings these services up but **not** the Okuna API itself, you are to run the API locally instead for a better development experience. #### Installation @@ -231,7 +231,7 @@ If the process was abruptly terminated and Okuna is still running in the backgro ```bash python okuna-cli.py down-services-only -``` +``` #### Running the Okuna API server locally @@ -286,8 +286,8 @@ Use this if you want to get a fresh version of Okuna next time you use the Okuna ```bash python okuna-cli.py clean -``` - +``` + ### Okuna CLI behind the scenes This section will try to demistify what the Okuna CLI does. @@ -319,7 +319,7 @@ When starting okuna-cli for the first time, 3 files will be generated A list of official and custom django commands that might come handy. -If running the API locally you can execute them as +If running the API locally you can execute them as ```bash python manage.py $command @@ -332,7 +332,7 @@ machine by running docker-compose -f docker-compose-full.yml exec webserver "/bin/bash" ``` -Inside the machine you can then execute the commands as +Inside the machine you can then execute the commands as ```bash python manage.py $command @@ -394,7 +394,7 @@ usage: manage.py reset_invite_email_boolean [-h] [--days DAYS] #### `manage.py send_invites` -Send invite emails to all user invites who have not been sent the email. +Send invite emails to all user invites who have not been sent the email. ```bash usage: manage.py send_invites [-h] @@ -402,7 +402,7 @@ usage: manage.py send_invites [-h] ####`manage.py allocate_invites` -Assign user invites to all or specific users. +Assign user invites to all or specific users. ```bash usage: manage.py allocate_invites [-h] [--count INCREMENT_INVITES_BY_COUNT --limit [INVITE_COUNT_UPPER_LIMIT]] [--total TOTAL_INVITE_COUNT_TO_SET] [--username USERNAME] @@ -449,7 +449,7 @@ usage: manage.py flush_proxy_blacklisted_domains #### `manage.py worker_health_check` -A a Django management command available for checking the worker health: +A a Django management command available for checking the worker health: Each queue has a required configurable `treshold`. These are configured in the Django settings. The `FAILED_JOB_THRESHOLD` is the maximum amount of failed jobs that are allowed, before an alert is sent using the `openbook_common.helpers.send_alert_to_channel` command, which sends an alert to a monitoring channel on i.e. Slack. Using the `ALERT_HOOK_URL` option in the Django settings file, it is possible to add the Slack hook URL. @@ -462,7 +462,7 @@ It is recommended to schedule the worker monitoring functions, to run at a 5 min #### Crowdin translations update -Download the latest django.po files in the respective locale/ folders from crowdin. +Download the latest django.po files in the respective locale/ folders from crowdin. Then locally run all or some of these commands depending on which models need updating. It will update the `.json` files, then check them in. ```$xslt @@ -474,8 +474,6 @@ It will update the `.json` files, then check them in. ## Available Django jobs -To schedule a job, go to the `/admin/scheduler` route on the running webserver. - The available jobs are ### openbook_posts.jobs.flush_draft_posts @@ -509,13 +507,13 @@ source strings and a place to enter the translation strings. It doesnt overwrite 4. If you find a fuzzy string, you can resolve it manually. Finally each string should like this ``` #: openbook_lists/validators.py:10 <- place where the string occurs in code -msgid "The list does not exist." <- english translations +msgid "The list does not exist." <- english translations msgstr "Die Liste ist nicht vorhanden." <-- this will be empty for new strings ``` 5. Upload this `django.po` file to https://crowdin.com/project/okuna/settings#files by pressing `Update` next to the existing `django.po` file. -6. Once all language volunteers have translated the new strings, download all the `django.po` files for each locale and +6. Once all language volunteers have translated the new strings, download all the `django.po` files for each locale and put them in their respective folders. -7. Run `./manage.py compilemessages` to auto-generate `django.mo` files. +7. Run `./manage.py compilemessages` to auto-generate `django.mo` files. 8. You need to checkin both `django.po` and `django.mo` files for each locale. @@ -531,4 +529,3 @@ python manage.py runserver --noreload #### Happy coding 🎉! - diff --git a/openbook/__init__.py b/openbook/__init__.py index e69de29b..067f4d3b 100644 --- a/openbook/__init__.py +++ b/openbook/__init__.py @@ -0,0 +1,3 @@ +from .celery import celery as celery_app + +__all__ = ('celery_app',) diff --git a/openbook/celery.py b/openbook/celery.py new file mode 100644 index 00000000..e2767631 --- /dev/null +++ b/openbook/celery.py @@ -0,0 +1,77 @@ +from datetime import timedelta +import os +from functools import wraps + +from celery import Celery +from kombu import Queue, Exchange +from openbook.settings import CELERY_REDIS_BROKER_LOCATION, CELERY_REDIS_RESULT_BACKEND_LOCATION + +CELERY_DEFAULT_PRIORITY_QUEUE = 'default_priority' +CELERY_LOW_PRIORITY_QUEUE = 'low_priority' +CELERY_HIGH_PRIORITY_QUEUE = 'high_priority' + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'openbook.settings') + +celery = Celery( + 'openbook', + broker=CELERY_REDIS_BROKER_LOCATION, + backend=CELERY_REDIS_RESULT_BACKEND_LOCATION, +) + +celery.conf['task_queues'] = ( + Queue( + CELERY_DEFAULT_PRIORITY_QUEUE, + Exchange(CELERY_DEFAULT_PRIORITY_QUEUE), + routing_key=CELERY_DEFAULT_PRIORITY_QUEUE + ), + Queue( + CELERY_LOW_PRIORITY_QUEUE, + Exchange(CELERY_LOW_PRIORITY_QUEUE), + routing_key=CELERY_LOW_PRIORITY_QUEUE + ), + Queue( + CELERY_HIGH_PRIORITY_QUEUE, + Exchange(CELERY_HIGH_PRIORITY_QUEUE), + routing_key=CELERY_HIGH_PRIORITY_QUEUE + ), +) + +# TODO: make job scheduler dynamically configurable +celery.conf['beat_schedule'] = { + 'flush-draft-posts': { + 'task': 'openbook_posts.jobs.flush_draft_posts', + 'schedule': timedelta(hours=1), + }, + + 'curate-top-posts': { + 'task': 'openbook_posts.jobs.curate_top_posts', + 'schedule': timedelta(minutes=15), + }, + + 'clean-top-posts': { + 'task': 'openbook_posts.jobs.clean_top_posts', + 'schedule': timedelta(hours=12), + }, + + 'curate-trending-posts': { + 'task': 'openbook_posts.jobs.curate_trending_posts', + 'schedule': timedelta(minutes=5), + }, + + 'clean-trending-posts': { + 'task': 'openbook_posts.jobs.clean_trending_posts', + 'schedule': timedelta(hours=1), + }, +} + +def celery_use_eager(func): + """ + Decorator that makes sure that the tasks invoked by a function are executed + in eager mode. Useful for unit testing. + """ + @wraps(func) + def inner(*args, **kwargs): + celery.conf.update(task_always_eager=True) + func(*args, **kwargs) + celery.conf.update(task_always_eager=False) + return inner diff --git a/openbook/settings.py b/openbook/settings.py index 09872d78..c22a32fe 100644 --- a/openbook/settings.py +++ b/openbook/settings.py @@ -21,7 +21,7 @@ from django_replicated.settings import * # Logging config -from sentry_sdk.integrations.rq import RqIntegration +from sentry_sdk.integrations.celery import CeleryIntegration from openbook_common.utils.environment import EnvironmentChecker @@ -114,8 +114,6 @@ 'imagekit', 'django_media_fixtures', 'cacheops', - 'django_rq', - 'scheduler', 'django_extensions', 'openbook_common', 'openbook_auth', @@ -171,12 +169,10 @@ 'host': REDIS_HOST, 'port': REDIS_PORT} -RQ_SHOW_ADMIN_LINK = False - REDIS_DEFAULT_CACHE_LOCATION = '%(redis_location)s/%(db)d' % {'redis_location': REDIS_LOCATION, 'db': 0} -REDIS_RQ_DEFAULT_JOBS_CACHE_LOCATION = '%(redis_location)s/%(db)d' % {'redis_location': REDIS_LOCATION, 'db': 1} -REDIS_RQ_HIGH_JOBS_CACHE_LOCATION = '%(redis_location)s/%(db)d' % {'redis_location': REDIS_LOCATION, 'db': 2} -REDIS_RQ_LOW_JOBS_CACHE_LOCATION = '%(redis_location)s/%(db)d' % {'redis_location': REDIS_LOCATION, 'db': 3} + +CELERY_REDIS_BROKER_LOCATION = '%(redis_location)s/%(db)d' % {'redis_location': REDIS_LOCATION, 'db': 1} +CELERY_REDIS_RESULT_BACKEND_LOCATION = '%(redis_location)s/%(db)d' % {'redis_location': REDIS_LOCATION, 'db': 2} CACHES = { 'default': { @@ -187,30 +183,6 @@ }, "KEY_PREFIX": "ob-api-" }, - 'rq-default-jobs': { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": REDIS_RQ_DEFAULT_JOBS_CACHE_LOCATION, - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient" - }, - "KEY_PREFIX": "ob-api-rq-default-job-" - }, - 'rq-high-jobs': { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": REDIS_RQ_HIGH_JOBS_CACHE_LOCATION, - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient" - }, - "KEY_PREFIX": "ob-api-rq-high-job-" - }, - 'rq-low-jobs': { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": REDIS_RQ_LOW_JOBS_CACHE_LOCATION, - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient" - }, - "KEY_PREFIX": "ob-api-rq-low-job-" - }, } CACHEOPS_REDIS_DB = int(os.environ.get('CACHEOPS_REDIS_DB', '1')) @@ -226,18 +198,6 @@ '*.*': {}, } -RQ_QUEUES = { - 'default': { - 'USE_REDIS_CACHE': 'rq-default-jobs', - }, - 'high': { - 'USE_REDIS_CACHE': 'rq-high-jobs', - }, - 'low': { - 'USE_REDIS_CACHE': 'rq-low-jobs', - }, -} - if IS_BUILD: NOSE_ARGS = [ '--cover-erase', @@ -393,7 +353,7 @@ raise NameError('SENTRY_DSN environment variable is required when running on a production environment') sentry_sdk.init( dsn=SENTRY_DSN, - integrations=[DjangoIntegration(), RqIntegration()] + integrations=[DjangoIntegration(), CeleryIntegration()] ) else: if SENTRY_DSN: diff --git a/openbook/urls.py b/openbook/urls.py index da28ff46..62ae18c6 100644 --- a/openbook/urls.py +++ b/openbook/urls.py @@ -460,7 +460,6 @@ urlpatterns = [ path('api/', include(api_patterns)), url('admin/', admin.site.urls), - path('django-rq/', include('django_rq.urls')), url('health/', Health.as_view(), name='health'), ] diff --git a/openbook_common/management/commands/worker_health_check.py b/openbook_common/management/commands/worker_health_check.py index 524aab9c..5116c1c2 100644 --- a/openbook_common/management/commands/worker_health_check.py +++ b/openbook_common/management/commands/worker_health_check.py @@ -1,3 +1,5 @@ +# TODO: port this over to Celery + from sys import exit from logging import getLogger diff --git a/openbook_common/utils/rq_helpers.py b/openbook_common/utils/rq_helpers.py index e22287b3..937da7ae 100644 --- a/openbook_common/utils/rq_helpers.py +++ b/openbook_common/utils/rq_helpers.py @@ -1,3 +1,5 @@ +# TODO: port this over to Celery + import django_rq from django_rq.utils import get_statistics, FailedJobRegistry diff --git a/openbook_notifications/helpers.py b/openbook_notifications/helpers.py index e7f958d2..c94e10f3 100644 --- a/openbook_notifications/helpers.py +++ b/openbook_notifications/helpers.py @@ -1,10 +1,9 @@ -import django_rq from django.utils import translation from django.utils.translation import ugettext_lazy as _ import onesignal as onesignal_sdk from openbook_common.utils.model_loaders import get_notification_model -from openbook_notifications.django_rq_jobs import send_notification_to_user_with_id +from openbook_notifications.jobs import send_notification_to_user_with_id from openbook_translation import translation_strategy import logging diff --git a/openbook_notifications/django_rq_jobs.py b/openbook_notifications/jobs.py similarity index 93% rename from openbook_notifications/django_rq_jobs.py rename to openbook_notifications/jobs.py index 1ad440ab..b0338ee4 100644 --- a/openbook_notifications/django_rq_jobs.py +++ b/openbook_notifications/jobs.py @@ -1,8 +1,8 @@ from hashlib import sha256 import onesignal as onesignal_sdk from django.conf import settings -from django_rq import job +from openbook.celery import celery from openbook_common.utils.model_loaders import get_user_model onesignal_client = onesignal_sdk.Client( @@ -10,8 +10,7 @@ app_auth_key=settings.ONE_SIGNAL_API_KEY ) - -@job('default') +@celery.task(queue='default_priority') def send_notification_to_user_with_id(user_id, notification): User = get_user_model() user = User.objects.only('username', 'uuid', 'id').get(pk=user_id) diff --git a/openbook_posts/jobs.py b/openbook_posts/jobs.py index 2032d009..9726fb7f 100644 --- a/openbook_posts/jobs.py +++ b/openbook_posts/jobs.py @@ -1,19 +1,18 @@ from django.utils import timezone -from django_rq import job from video_encoding import tasks from datetime import timedelta from django.db.models import Q, Count from django.conf import settings from cursor_pagination import CursorPaginator +from openbook.celery import celery from openbook_common.utils.model_loaders import get_post_model, get_post_media_model, get_community_model, \ get_top_post_model, get_post_comment_model, get_moderated_object_model, get_trending_post_model import logging logger = logging.getLogger(__name__) - -@job('low') +@celery.task(queue='low_priority') def flush_draft_posts(): """ This job should be scheduled to get all pending draft posts for a day and remove them @@ -32,7 +31,7 @@ def flush_draft_posts(): return 'Flushed %s posts' % str(flushed_posts) -@job('high') +@celery.task(queue='high_priority') def process_post_media(post_id): """ This job is called to process post media and mark it as published @@ -53,7 +52,7 @@ def process_post_media(post_id): logger.info('Processed media of post with id: %d' % post_id) -@job('low') +@celery.task(queue='low_priority') def curate_top_posts(): """ Curates the top posts. @@ -119,7 +118,7 @@ def curate_top_posts(): return 'Checked: %d. Curated: %d' % (total_checked_posts, total_curated_posts) -@job('low') +@celery.task(queue='low_priority') def clean_top_posts(): """ Cleans up top posts, that no longer meet the criteria. @@ -198,7 +197,7 @@ def _add_post_to_top_post(post): return None -@job('low') +@celery.task(queue='low_priority') def curate_trending_posts(): """ Curates the trending posts. @@ -249,7 +248,7 @@ def curate_trending_posts(): return 'Curated: %d posts' % posts.count() -@job('low') +@celery.task(queue='low_priority') def bootstrap_trending_posts(): """ Bootstraps the trending posts. @@ -303,7 +302,7 @@ def bootstrap_trending_posts(): return 'Checked: %d. Curated: %d' % (total_checked_posts, total_curated_posts) -@job('low') +@celery.task(queue='low_priority') def clean_trending_posts(): """ Cleans trending posts. diff --git a/openbook_posts/tests/views/test_post.py b/openbook_posts/tests/views/test_post.py index 1aebf7d8..44ad16ca 100644 --- a/openbook_posts/tests/views/test_post.py +++ b/openbook_posts/tests/views/test_post.py @@ -5,8 +5,6 @@ from PIL import Image from django.urls import reverse -from django_rq import get_worker -from django_rq.queues import get_queues from faker import Faker from rest_framework import status from openbook_common.tests.models import OpenbookAPITestCase @@ -18,7 +16,7 @@ import logging -from rq import SimpleWorker, Worker +from openbook.celery import celery_use_eager from openbook_common.tests.helpers import make_authentication_headers_for_user, make_fake_post_text, \ make_fake_post_comment_text, make_user, make_circle, make_community, make_moderation_category, \ @@ -655,6 +653,7 @@ def test_delete_image_post(self): self.assertFalse(access(file.name, F_OK)) + @celery_use_eager def test_delete_video_post(self): """ should be able to delete video post and file @@ -668,9 +667,6 @@ def test_delete_video_post(self): post = user.create_public_post(text=make_fake_post_text(), video=video) - # Process videos - get_worker('high', worker_class=SimpleWorker).work(burst=True) - post_media_video = post.get_first_media() post_video = post_media_video.content_object @@ -2728,6 +2724,7 @@ class PublishPostAPITests(OpenbookAPITestCase): 'openbook_circles/fixtures/circles.json' ] + @celery_use_eager def test_publishing_draft_image_post_should_process_media(self): """ should process draft image post when publishing @@ -2751,15 +2748,13 @@ def test_publishing_draft_image_post_should_process_media(self): post = Post.objects.get(pk=post.pk) - self.assertEqual(post.status, Post.STATUS_PROCESSING) - - # Run the process handled by a worker - get_worker('high', worker_class=SimpleWorker).work(burst=True) - - post.refresh_from_db() + # TODO: wait for Celery task manually (without eager mode) + # self.assertEqual(post.status, Post.STATUS_PROCESSING) + # post.refresh_from_db() self.assertEqual(post.status, Post.STATUS_PUBLISHED) + @celery_use_eager def test_publishing_draft_video_post_should_process_media(self): """ should process draft video post mp4|3gp|gif media when publishing @@ -2782,12 +2777,9 @@ def test_publishing_draft_video_post_should_process_media(self): post = Post.objects.get(pk=post.pk) - self.assertEqual(post.status, Post.STATUS_PROCESSING) - - # Run the process handled by a worker - get_worker('high', worker_class=SimpleWorker).work(burst=True) - - post.refresh_from_db() + # TODO: wait for Celery task manually (without eager mode) + # self.assertEqual(post.status, Post.STATUS_PROCESSING) + # post.refresh_from_db() self.assertEqual(post.status, Post.STATUS_PUBLISHED) @@ -2924,6 +2916,7 @@ def test_cant_publish_foreign_user_draft_encircled_post(self): self.assertTrue(Post.objects.filter(pk=post.pk, status=Post.STATUS_DRAFT)) + @celery_use_eager def test_publishing_publicly_visible_image_post_with_new_hashtag_should_use_image(self): """ when publishing a publicly visible post with a new hashtag, the hashtag should use the image @@ -2951,12 +2944,9 @@ def test_publishing_publicly_visible_image_post_with_new_hashtag_should_use_imag post = Post.objects.get(pk=post.pk) - self.assertEqual(post.status, Post.STATUS_PROCESSING) - - # Run the process handled by a worker - get_worker('high', worker_class=SimpleWorker).work(burst=True) - - post.refresh_from_db() + # TODO: wait for Celery task manually (without eager mode) + # self.assertEqual(post.status, Post.STATUS_PROCESSING) + # post.refresh_from_db() self.assertEqual(post.status, Post.STATUS_PUBLISHED) diff --git a/openbook_posts/tests/views/test_post_media.py b/openbook_posts/tests/views/test_post_media.py index 25dd7524..b356c870 100644 --- a/openbook_posts/tests/views/test_post_media.py +++ b/openbook_posts/tests/views/test_post_media.py @@ -6,10 +6,10 @@ from django.conf import settings from django.core.files import File from django.urls import reverse -from django_rq import get_worker from faker import Faker from rest_framework import status -from rq import SimpleWorker + +from openbook.celery import celery_use_eager from openbook_common.tests.models import OpenbookAPITestCase import random @@ -182,6 +182,7 @@ def test_cannot_add_media_to_draft_if_exceeds_settings_maximum(self): self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + @celery_use_eager def test_add_first_media_video_creates_media_thumbnail_and_dimensions(self): """ should create a post media_thumbnail and dimensions when adding the first media video @@ -302,6 +303,7 @@ def test_can_retrieve_post_empty_media_if_no_media(self): self.assertEqual(len(parsed_response), 0) + @celery_use_eager def test_can_retrieve_own_post_media_image(self): """ should be able to retrieve an own post media image @@ -316,8 +318,6 @@ def test_can_retrieve_own_post_media_image(self): file = File(file) post = user.create_public_post(image=file) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') @@ -332,6 +332,7 @@ def test_can_retrieve_own_post_media_image(self): self._compare_response_media_with_post_media(post_media=post_media, response_media=response_media) + @celery_use_eager def test_can_retrieve_own_post_media_video(self): """ should be able to retrieve an own post media video @@ -346,8 +347,6 @@ def test_can_retrieve_own_post_media_video(self): file = File(file) post = user.create_public_post(video=file) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') @@ -364,6 +363,7 @@ def test_can_retrieve_own_post_media_video(self): # foreign + @celery_use_eager def test_can_retrieve_foreign_user_post_media_image(self): """ should be able to retrieve an foreign_user post media image @@ -379,8 +379,6 @@ def test_can_retrieve_foreign_user_post_media_image(self): file = File(file) post = foreign_user.create_public_post(image=file) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') @@ -395,6 +393,7 @@ def test_can_retrieve_foreign_user_post_media_image(self): self._compare_response_media_with_post_media(post_media=post_media, response_media=response_media) + @celery_use_eager def test_can_retrieve_foreign_user_post_media_video(self): """ should be able to retrieve an foreign_user post media video @@ -410,8 +409,6 @@ def test_can_retrieve_foreign_user_post_media_video(self): file = File(file) post = foreign_user.create_public_post(video=file) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') @@ -426,6 +423,7 @@ def test_can_retrieve_foreign_user_post_media_video(self): self._compare_response_media_with_post_media(post_media=post_media, response_media=response_media) + @celery_use_eager def test_can_retrieve_following_user_post_media_image(self): """ should be able to retrieve an following_user post media image @@ -443,8 +441,6 @@ def test_can_retrieve_following_user_post_media_image(self): file = File(file) post = following_user.create_public_post(image=file) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') @@ -459,6 +455,7 @@ def test_can_retrieve_following_user_post_media_image(self): self._compare_response_media_with_post_media(post_media=post_media, response_media=response_media) + @celery_use_eager def test_can_retrieve_following_user_post_media_video(self): """ should be able to retrieve an following_user post media video @@ -476,8 +473,6 @@ def test_can_retrieve_following_user_post_media_video(self): file = File(file) post = following_user.create_public_post(video=file) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') @@ -492,6 +487,7 @@ def test_can_retrieve_following_user_post_media_video(self): self._compare_response_media_with_post_media(post_media=post_media, response_media=response_media) + @celery_use_eager def test_can_retrieve_follower_user_post_media_image(self): """ should be able to retrieve an follower_user post media image @@ -509,8 +505,6 @@ def test_can_retrieve_follower_user_post_media_image(self): file = File(file) post = follower_user.create_public_post(image=file) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') @@ -525,6 +519,7 @@ def test_can_retrieve_follower_user_post_media_image(self): self._compare_response_media_with_post_media(post_media=post_media, response_media=response_media) + @celery_use_eager def test_can_retrieve_follower_user_post_media_video(self): """ should be able to retrieve an follower_user post media video @@ -542,8 +537,6 @@ def test_can_retrieve_follower_user_post_media_video(self): file = File(file) post = follower_user.create_public_post(video=file) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') @@ -558,6 +551,7 @@ def test_can_retrieve_follower_user_post_media_video(self): self._compare_response_media_with_post_media(post_media=post_media, response_media=response_media) + @celery_use_eager def test_can_retrieve_connected_user_post_media_image(self): """ should be able to retrieve an connected_user post media image @@ -577,8 +571,6 @@ def test_can_retrieve_connected_user_post_media_image(self): file = File(file) post = connected_user.create_encircled_post(image=file, circles_ids=[circle.pk]) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') @@ -593,6 +585,7 @@ def test_can_retrieve_connected_user_post_media_image(self): self._compare_response_media_with_post_media(post_media=post_media, response_media=response_media) + @celery_use_eager def test_can_retrieve_connected_user_post_media_video(self): """ should be able to retrieve an connected_user post media video @@ -613,8 +606,6 @@ def test_can_retrieve_connected_user_post_media_video(self): file = File(file) post = connected_user.create_encircled_post(video=file, circles_ids=[circle.pk]) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') @@ -629,6 +620,7 @@ def test_can_retrieve_connected_user_post_media_video(self): self._compare_response_media_with_post_media(post_media=post_media, response_media=response_media) + @celery_use_eager def test_cant_retrieve_pending_connection_user_user_encircled_post_media_image(self): """ should not be able to retrieve an pending_connection_user_user encircled post media image @@ -647,14 +639,13 @@ def test_cant_retrieve_pending_connection_user_user_encircled_post_media_image(s circle = make_circle(creator=pending_connection_user_user) post = pending_connection_user_user.create_encircled_post(image=file, circles_ids=[circle.pk]) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + @celery_use_eager def test_cant_retrieve_pending_connection_user_user_encircled_post_media_video(self): """ should be able to retrieve an pending_connection_user_user encircled post media video @@ -673,14 +664,13 @@ def test_cant_retrieve_pending_connection_user_user_encircled_post_media_video(s circle = make_circle(creator=pending_connection_user_user) post = pending_connection_user_user.create_encircled_post(video=file, circles_ids=[circle.pk]) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + @celery_use_eager def test_can_retrieve_public_community_post_media_image(self): """ should be able to retrieve an public_community post media image @@ -699,8 +689,6 @@ def test_can_retrieve_public_community_post_media_image(self): file = File(file) post = community_member.create_community_post(image=file, community_name=public_community.name) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') @@ -715,6 +703,7 @@ def test_can_retrieve_public_community_post_media_image(self): self._compare_response_media_with_post_media(post_media=post_media, response_media=response_media) + @celery_use_eager def test_can_retrieve_public_community_post_media_video(self): """ should be able to retrieve an public_community post media video @@ -733,8 +722,6 @@ def test_can_retrieve_public_community_post_media_video(self): file = File(file) post = community_member.create_community_post(video=file, community_name=public_community.name) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') @@ -749,6 +736,7 @@ def test_can_retrieve_public_community_post_media_video(self): self._compare_response_media_with_post_media(post_media=post_media, response_media=response_media) + @celery_use_eager def test_can_retrieve_private_community_part_of_post_media_image(self): """ should be able to retrieve an private_community part of post media image @@ -769,8 +757,6 @@ def test_can_retrieve_private_community_part_of_post_media_image(self): file = File(file) post = community_creator.create_community_post(image=file, community_name=private_community.name) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') @@ -785,6 +771,7 @@ def test_can_retrieve_private_community_part_of_post_media_image(self): self._compare_response_media_with_post_media(post_media=post_media, response_media=response_media) + @celery_use_eager def test_can_retrieve_private_community_part_of_post_media_video(self): """ should be able to retrieve an private_community part of post media video @@ -805,8 +792,6 @@ def test_can_retrieve_private_community_part_of_post_media_video(self): file = File(file) post = community_creator.create_community_post(video=file, community_name=private_community.name) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') @@ -821,6 +806,7 @@ def test_can_retrieve_private_community_part_of_post_media_video(self): self._compare_response_media_with_post_media(post_media=post_media, response_media=response_media) + @celery_use_eager def test_cannot_retrieve_private_community_not_part_of_post_media_image(self): """ should not be able to retrieve an private_community not part of post media image @@ -837,14 +823,13 @@ def test_cannot_retrieve_private_community_not_part_of_post_media_image(self): file = File(file) post = community_creator.create_community_post(image=file, community_name=private_community.name) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + @celery_use_eager def test_cannot_retrieve_private_community_not_part_of_post_media_video(self): """ should not be able to retrieve an private_community not part of post media video @@ -861,8 +846,6 @@ def test_cannot_retrieve_private_community_not_part_of_post_media_video(self): file = File(file) post = community_creator.create_community_post(video=file, community_name=private_community.name) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - url = self._get_url(post=post) response = self.client.get(url, **headers, format='multipart') diff --git a/openbook_posts/tests/views/test_posts.py b/openbook_posts/tests/views/test_posts.py index d164b4b2..2029d9df 100644 --- a/openbook_posts/tests/views/test_posts.py +++ b/openbook_posts/tests/views/test_posts.py @@ -7,10 +7,8 @@ from django.core.files import File from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse -from django_rq import get_worker from faker import Faker from rest_framework import status -from rq import SimpleWorker from openbook_common.tests.models import OpenbookAPITestCase from mixer.backend.django import mixer @@ -22,6 +20,8 @@ import logging import json +from openbook.celery import celery_use_eager + from openbook_circles.models import Circle from openbook_common.tests.helpers import make_user, make_users, make_fake_post_text, \ make_authentication_headers_for_user, make_circle, make_community, make_list, make_moderation_category, \ @@ -369,6 +369,7 @@ def test_create_text_detect_mention_is_case_insensitive(self): self.assertTrue(PostUserMention.objects.filter(post_id=post.pk, user_id=mentioned_user.pk).exists()) + @celery_use_eager def test_create_text_detect_mention_ignores_casing_of_username(self): """ should detect mention regardless of the casing of the username @@ -388,8 +389,6 @@ def test_create_text_detect_mention_ignores_casing_of_username(self): url = self._get_url() response = self.client.put(url, data, **headers, format='multipart') - get_worker('high', worker_class=SimpleWorker).work(burst=True) - post = Post.objects.get(text=post_text, creator_id=user.pk) self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertTrue(PostUserMention.objects.filter(post_id=post.pk, user_id=mentioned_user.pk).exists()) @@ -836,6 +835,7 @@ def test_create_image_post_creates_hash(self): media = PostMedia.objects.get(post_id=response_post_id, type=PostMedia.MEDIA_TYPE_IMAGE) self.assertEqual(media.content_object.hash, filehash) + @celery_use_eager def test_create_video_post(self): """ should be able to create a video post and return 201 @@ -868,8 +868,6 @@ def test_create_video_post(self): self.assertTrue(created_post.status, Post.STATUS_PROCESSING) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - created_post.refresh_from_db() self.assertTrue(created_post.status, Post.STATUS_PUBLISHED) @@ -935,6 +933,7 @@ def test_create_video_post_creates_thumbnail(self): self.assertIsNotNone(media.content_object.thumbnail_width) self.assertIsNotNone(media.content_object.thumbnail_height) + @celery_use_eager def test_create_video_and_text_post(self): """ should be able to create a video and text post and return 201 @@ -971,8 +970,6 @@ def test_create_video_and_text_post(self): self.assertEqual(created_post.text, post_text) - get_worker('high', worker_class=SimpleWorker).work(burst=True) - created_post.refresh_from_db() self.assertTrue(created_post.status, Post.STATUS_PUBLISHED) diff --git a/requirements.txt b/requirements.txt index 1bdaad02..ff627454 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ bandit==1.5.1 beautifulsoup4==4.9.3 boto3==1.16.0 botocore==1.19.0 +celery==5.1.2 certifi==2020.6.20 chardet==3.0.4 click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' From 0daf58c8dca2c37650f354646f4a710e03136d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B3zsef=20Sallai?= Date: Tue, 17 Aug 2021 22:45:17 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=90=B3=20reconfigure=20docker=20conta?= =?UTF-8?q?iners=20to=20use=20celery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .docker/scheduler/supervisord.conf | 32 +-------- .docker/worker/supervisord.conf | 14 ++-- .gitignore | 1 + Pipfile | 1 - Pipfile.lock | 101 +++++++++++------------------ docker-compose-full.yml | 2 +- docker-compose-services-only.yml | 3 +- requirements.txt | 4 -- 8 files changed, 50 insertions(+), 108 deletions(-) diff --git a/.docker/scheduler/supervisord.conf b/.docker/scheduler/supervisord.conf index a7057d12..229126ea 100644 --- a/.docker/scheduler/supervisord.conf +++ b/.docker/scheduler/supervisord.conf @@ -9,8 +9,8 @@ serverurl = unix: username=admin password=revproxy -[program: rqschedulerdefault] -command = python manage.py rqscheduler --queue=default +[program: celerybeat] +command = celery -A openbook beat -l info loglevel = info ; (log level;default info; others: debug, warn, trace) numprocs = 1 directory = /opt/okuna-api @@ -22,31 +22,3 @@ exitcodes = 0, 2 ; 'expected' exit codes for process (default 0, killasgroup = true ; SIGKILL the UNIX process group (def false) stopasgroup = true stopsignal = QUIT - -[program: rqschedulerhigh] -command = python manage.py rqscheduler --queue=high -loglevel = info ; (log level;default info; others: debug, warn, trace) -numprocs = 1 -directory = /opt/okuna-api -autostart = true -autorestart = unexpected -startsecs = 2 ; number of secs prog must stay running (def. 1) -startretries = 3 ; max # of serial start failures (default 3) -exitcodes = 0, 2 ; 'expected' exit codes for process (default 0, 2) -killasgroup = true ; SIGKILL the UNIX process group (def false) -stopasgroup = true -stopsignal = QUIT - -[program: rqschedulerlow] -command = python manage.py rqscheduler --queue=low -loglevel = info ; (log level;default info; others: debug, warn, trace) -numprocs = 1 -directory = /opt/okuna-api -autostart = true -autorestart = unexpected -startsecs = 1 ; number of secs prog must stay running (def. 1) -startretries = 3 ; max # of serial start failures (default 3) -exitcodes = 0, 2 ; 'expected' exit codes for process (default 0, 2) -killasgroup = true ; SIGKILL the UNIX process group (def false) -stopasgroup = true -stopsignal = QUIT \ No newline at end of file diff --git a/.docker/worker/supervisord.conf b/.docker/worker/supervisord.conf index 6b5be544..081192ab 100644 --- a/.docker/worker/supervisord.conf +++ b/.docker/worker/supervisord.conf @@ -9,8 +9,8 @@ serverurl = unix: username=admin password=revproxy -[program: rqworkerdefault] -command = python manage.py rqworker default +[program: celerydefault] +command = celery -A openbook worker -E -l info -Q default_priority -n okuna_default loglevel = info ; (log level;default info; others: debug, warn, trace) numprocs = 1 directory = /opt/okuna-api @@ -23,8 +23,8 @@ killasgroup = true ; SIGKILL the UNIX process group (def false) stopasgroup = true stopsignal = QUIT -[program: rqworkerhigh] -command = python manage.py rqworker high +[program: celeryhigh] +command = celery -A openbook worker -E -l info -Q high_priority -n okuna_high loglevel = info ; (log level;default info; others: debug, warn, trace) numprocs = 1 directory = /opt/okuna-api @@ -37,8 +37,8 @@ killasgroup = true ; SIGKILL the UNIX process group (def false) stopasgroup = true stopsignal = QUIT -[program: rqworkerlow] -command = python manage.py rqworker low +[program: celerylow] +command = celery -A openbook worker -E -l info -Q low_priority -n okuna_low loglevel = info ; (log level;default info; others: debug, warn, trace) numprocs = 1 directory = /opt/okuna-api @@ -49,4 +49,4 @@ startretries = 3 ; max # of serial start failures (default 3) exitcodes = 0, 2 ; 'expected' exit codes for process (default 0, 2) killasgroup = true ; SIGKILL the UNIX process group (def false) stopasgroup = true -stopsignal = QUIT \ No newline at end of file +stopsignal = QUIT diff --git a/.gitignore b/.gitignore index a80ba220..f50574c9 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ src/ static/ docker-compose.override.yml *.db +celerybeat-schedule # Elastic Beanstalk Files .elasticbeanstalk/* diff --git a/Pipfile b/Pipfile index 70b640f5..1f8732d7 100644 --- a/Pipfile +++ b/Pipfile @@ -40,7 +40,6 @@ django-redis = "*" django-extensions = "*" langdetect = "*" redis = "*" -django-rq-scheduler = "*" ffmpy = "*" django-positions = "*" django-ordered-model = "*" diff --git a/Pipfile.lock b/Pipfile.lock index f5546002..9f1786c8 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "761e5c228d2f131fb1a1bc84764e2fe8b1b2aaae5476e4e714cb42aa7d2b0b7e" + "sha256": "612e9a386960ec42224b21565321e8e0d329af06b5f5253f656c995d5dde3720" }, "pipfile-spec": 6, "requires": {}, @@ -19,6 +19,7 @@ "sha256:03e16e94f2b34c31f8bf1206d8ddd3ccaa4c315f7f6a1879b7b1210d229568c2", "sha256:493a2ac6788ce270a2f6a765b017299f60c1998f5a8617908ee9be082f7300fb" ], + "markers": "python_version >= '3.6'", "version": "==5.0.6" }, "appdirs": { @@ -54,18 +55,19 @@ }, "boto3": { "hashes": [ - "sha256:6cc7011cb857fecee54884ff344d6b793cd22af51142f715706c757d26d02bb1", - "sha256:7405ae77ce4f2151fae1b542183f9c0f7ffb57c288b1f152819cfcb88e9cf297" + "sha256:1b08ace99e7b92965780e5ce759430ad62b7b7e037560bc772f9a8789f4f36d2", + "sha256:31cc69e665f773390c4c17ce340d2420e45fbac51d46d945cc4a58d483ec5da6" ], "index": "pypi", - "version": "==1.18.22" + "version": "==1.18.23" }, "botocore": { "hashes": [ - "sha256:9c133caab58b04b4a9ab3f6523cc61cf815c1a5fde7b5ee279eefa48dc3a01d1", - "sha256:9df7a84840bcea10eb68f816d562c77656ec253a3a0dc3724e7e9ac086656e28" + "sha256:3877d69e0b718b786f1696cd04ddbdb3a57aef6adb0239a29aa88754489849a4", + "sha256:d0146d31dbc475942b578b47dd5bcf94d18fbce8c6d2ce5f12195e005de9b754" ], - "version": "==1.21.22" + "markers": "python_version >= '3.6'", + "version": "==1.21.23" }, "celery": { "hashes": [ @@ -95,6 +97,7 @@ "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==7.1.2" }, "click-didyoumean": { @@ -122,6 +125,7 @@ "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==0.4.4" }, "colormath": { @@ -188,13 +192,6 @@ "index": "pypi", "version": "==5.5" }, - "croniter": { - "hashes": [ - "sha256:0f97b361fe343301a8f66f852e7d84e4fb7f21379948f71e1bbfe10f5d015fbd", - "sha256:a70dfc9d52de9fc1a886128b9148c89dd9e76b67d55f46516ca94d2d73d58219" - ], - "version": "==1.0.15" - }, "django": { "hashes": [ "sha256:62cf45e5ee425c52e411c0742e641a6588b7e8af0d2c274a27940931b2786594", @@ -257,13 +254,6 @@ "index": "pypi", "version": "==0.0.3" }, - "django-model-utils": { - "hashes": [ - "sha256:eb5dd05ef7d7ce6bc79cae54ea7c4a221f6f81e2aad7722933aee66489e7264b", - "sha256:ef7c440024e797796a3811432abdd2be8b5225ae64ef346f8bfc6de7d8e5d73c" - ], - "version": "==4.1.1" - }, "django-modeltranslation": { "hashes": [ "sha256:fdbb32edcf92b82c934b336c5fd0e0ecf68bd9ce56fc55b172a617edf9c252cd" @@ -317,21 +307,6 @@ "index": "pypi", "version": "==2.7" }, - "django-rq": { - "hashes": [ - "sha256:23981f83c537178cbbf730f192c13e99cede2204e9d917b1c1c80c42215dd227", - "sha256:f09059ab37403a47c7933bca396fabb7f3058732d132462eade5333bc4bcac5f" - ], - "version": "==2.4.1" - }, - "django-rq-scheduler": { - "hashes": [ - "sha256:4418e1397c5ce02ce1653f97ceb5ebf8b87902e072da81a88770a02b725d1461", - "sha256:8300d1cdc839fddd7723f4ff0c09089d6d182be0a6bdfed2357b6d8691010fd4" - ], - "index": "pypi", - "version": "==1.1.3" - }, "django-storages": { "hashes": [ "sha256:0a9b7e620e969fb0797523695329ed223bf540bbfdf6cd163b061fc11dab2d1c", @@ -353,6 +328,7 @@ "sha256:a1b5f169102e1c894f9a7d5ccf6f9402a836a5d24be80a986c7ce9eaed78f367", "sha256:e953a25e44ebb60a5c6efc2add4420c177f1d8404509da88da9729202f306994" ], + "markers": "python_version >= '3.5'", "version": "==0.5.1" }, "faker": { @@ -389,14 +365,16 @@ "sha256:6c4cc71933456991da20917998acbe6cf4fb41eeaab7d6d67fbc05ecd4c865b0", "sha256:96bf5c08b157a666fec41129e6d327235284cca4c81e92109260f353ba138005" ], + "markers": "python_version >= '3.4'", "version": "==4.0.7" }, "gitpython": { "hashes": [ - "sha256:b1e1c269deab1b08ce65403cf14e10d2ef1f6c89e33ea7c5e5bb0222ea593b8a", - "sha256:df0e072a200703a65387b0cfdf0466e3bab729c0458cf6b7349d0e9877636519" + "sha256:b838a895977b45ab6f0cc926a9045c8d1c44e2b653c1fcc39fe91f42c6e8f05b", + "sha256:fce760879cd2aebd2991b3542876dc5c4a909b30c9d69dfc488e504a8db37ee8" ], - "version": "==3.1.20" + "markers": "python_version >= '3.6'", + "version": "==3.1.18" }, "halo": { "hashes": [ @@ -411,6 +389,7 @@ "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a", "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3" ], + "markers": "python_version >= '3'", "version": "==3.2" }, "jmespath": { @@ -418,6 +397,7 @@ "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f" ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.0" }, "kombu": { @@ -425,6 +405,7 @@ "sha256:01481d99f4606f6939cdc9b637264ed353ee9e3e4f62cfb582324142c41a572d", "sha256:e2dedd8a86c9077c350555153825a31e456a0dc20c15d5751f00137ec9c75f0a" ], + "markers": "python_version >= '3.6'", "version": "==5.1.0" }, "langdetect": { @@ -466,6 +447,7 @@ "sha256:2306f1950ce772c5a59a57f5486d59bb9cab98497c45fc49cbc45ac0dec119bb", "sha256:5fcb7004be69e8fbdf07dcb502efa5c77cadcaad6982164134eeb9721f826c2e" ], + "markers": "python_version >= '3.7'", "version": "==2.6.2" }, "nose": { @@ -517,6 +499,7 @@ "sha256:f545c082eeb09ae678dd451a1b1dbf17babd8a0d7adea02897a76e639afca310", "sha256:fde50062d67d805bc96f1a9ecc0d37bfc2a8f02b937d2c50824d186aa91f2419" ], + "markers": "python_version < '3.11' and python_version >= '3.7'", "version": "==1.21.2" }, "onesignal-sdk": { @@ -532,6 +515,7 @@ "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7", "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14" ], + "markers": "python_version >= '3.6'", "version": "==21.0" }, "pbr": { @@ -539,6 +523,7 @@ "sha256:42df03e7797b796625b1029c0400279c7c34fd7df24a7d7818a1abb5b38710dd", "sha256:c68c661ac5cc81058ac94247278eeda6d2e6aecb3e227b0387c30d277e7ef8d4" ], + "markers": "python_version >= '2.6'", "version": "==5.6.0" }, "pilkit": { @@ -605,6 +590,7 @@ "sha256:08360ee3a3148bdb5163621709ee322ec34fc4375099afa4bbf751e9b7b7fa4f", "sha256:7089d8d2938043508aa9420ec18ce0922885304cddae87fb96eebca942299f88" ], + "markers": "python_full_version >= '3.6.1'", "version": "==3.0.19" }, "pyjwt": { @@ -620,6 +606,7 @@ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.4.7" }, "python-dateutil": { @@ -627,6 +614,7 @@ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.2" }, "python-dotenv": { @@ -700,6 +688,7 @@ "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24", "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==2.26.0" }, "requests-file": { @@ -717,25 +706,12 @@ "index": "pypi", "version": "==2.0.0" }, - "rq": { - "hashes": [ - "sha256:7af1e9706dbe6f1eac16dffacd8271ec27c1369950941f14dab6bb08a62979d7", - "sha256:bdfef943de838955e474cfd0e25b9b8c53ed4b9c361fe4bb11cf56d17a87acc5" - ], - "version": "==1.9.0" - }, - "rq-scheduler": { - "hashes": [ - "sha256:da94e9b6badf112995ff38fe16192e4f4c43c412b3c9614684ed8c8f7ca517d2", - "sha256:db79bb56cdbc4f7ffdd8bd659e389e91aa0db9c1abf002dc46f5dd6f0dbd2910" - ], - "version": "==0.11.0" - }, "s3transfer": { "hashes": [ "sha256:50ed823e1dc5868ad40c8dc92072f757aa0e653a192845c94a3b676f4a62da4c", "sha256:9c1dc369814391a6bda20ebbf4b70a0f34630592c9aa520856bf384916af2803" ], + "markers": "python_version >= '3.6'", "version": "==0.5.0" }, "safety": { @@ -766,6 +742,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "smmap": { @@ -773,6 +750,7 @@ "sha256:7e65386bd122d45405ddf795637b7f7d2b532e7e401d46bbe3fb49b9986d5182", "sha256:a9a7479e4c572e2e775c404dcd3080c8dc49f39918c2cf74913d30c4c478e3c2" ], + "markers": "python_version >= '3.5'", "version": "==4.0.0" }, "soupsieve": { @@ -780,7 +758,7 @@ "sha256:052774848f448cf19c7e959adf5566904d525f33a3f8b6ba6f6f8f26ec7de0cc", "sha256:c2c1c2d44f158cdbddab7824a9af8c4f83c76b1e23e049479aa432feb6c4c23b" ], - "markers": "python_version >= '3.0'", + "markers": "python_version >= '3'", "version": "==2.2.1" }, "spectra": { @@ -802,6 +780,7 @@ "sha256:017cde379adbd6a1f15a61873f43e8274179378e95ef3fede90b5aa64d304ed0", "sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8" ], + "markers": "python_version >= '3.5'", "version": "==0.4.1" }, "stevedore": { @@ -809,6 +788,7 @@ "sha256:3a5bbd0652bf552748871eaa73a4a8dc2899786bc497a2aa1fcb4dcdb0debeee", "sha256:50d7b78fbaf0d04cd62411188fa7eedcb03eb7f4c4b37005615ceebe582aa82a" ], + "markers": "python_version >= '3.6'", "version": "==3.3.0" }, "termcolor": { @@ -837,22 +817,15 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" }, - "typing-extensions": { - "hashes": [ - "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497", - "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342", - "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" - ], - "markers": "python_version < '3.10'", - "version": "==3.10.0.0" - }, "uritools": { "hashes": [ "sha256:28ffef82ce3b2793237d36e45aa7cde28dae6502f6a93fdbd05ede401520e279", "sha256:576737664f51f82d5c2a98e25f6c5da73a57cc88326dbb686fd6c5d06ebd6c29" ], + "markers": "python_version ~= '3.5'", "version": "==3.0.2" }, "url-normalize": { @@ -876,6 +849,7 @@ "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4", "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==1.26.6" }, "vine": { @@ -883,6 +857,7 @@ "sha256:4c9dceab6f76ed92105027c49c823800dd33cacce13bdedc5b914e3514b7fb30", "sha256:7d3b1624a953da82ef63462013bbd271d3eb75751489f9807598e8f340bd637e" ], + "markers": "python_version >= '3.6'", "version": "==5.0.0" }, "watchdog": { diff --git a/docker-compose-full.yml b/docker-compose-full.yml index c7044b7c..226b60a8 100644 --- a/docker-compose-full.yml +++ b/docker-compose-full.yml @@ -61,7 +61,7 @@ services: okuna: ipv4_address: 172.16.16.3 depends_on: - - webserver + - worker env_file: - .docker-compose.env db: diff --git a/docker-compose-services-only.yml b/docker-compose-services-only.yml index 15beabfa..b492a32b 100644 --- a/docker-compose-services-only.yml +++ b/docker-compose-services-only.yml @@ -39,8 +39,7 @@ services: okuna: ipv4_address: 172.16.16.3 depends_on: - - db - - redis + - worker env_file: - .docker-compose.env db: diff --git a/requirements.txt b/requirements.txt index ff627454..a2c713d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,8 +28,6 @@ django-positions==0.6.0 django-proxy==1.2.1 django-redis==4.12.1 django-replicated==2.7 -django-rq-scheduler==1.1.3 -django-rq==2.3.2 django-storages==1.8 django==2.2.16 djangorestframework==3.12.1 @@ -71,8 +69,6 @@ redis==3.5.3 requests-file==1.5.1 requests==2.24.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' rest-framework-generic-relations==2.0.0 -rq-scheduler==0.10.0 -rq==1.5.2; python_version >= '3.5' s3transfer==0.3.3 safety==1.9.0 sentry-sdk==0.10.2 From 603d723569fd03b61b60fb25f68f3fb5f6efaf38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B3zsef=20Sallai?= Date: Wed, 18 Aug 2021 14:25:07 +0300 Subject: [PATCH 3/5] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20use=20rabbitmq=20as=20?= =?UTF-8?q?a=20message=20broker=20instead=20of=20redis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- okuna-cli.py | 18 ++++++++++++++---- openbook/celery.py | 4 ++-- openbook/settings.py | 19 ++++++++++++++++++- templates/.docker-compose.env | 11 ++++++++++- templates/.env | 11 ++++++++++- templates/.okuna-cli.json | 5 +++-- 6 files changed, 57 insertions(+), 11 deletions(-) diff --git a/okuna-cli.py b/okuna-cli.py index ac29c938..3fa05d96 100755 --- a/okuna-cli.py +++ b/okuna-cli.py @@ -82,6 +82,10 @@ def _get_redis_password(): return _get_random_string(128) +def _get_rabbitmq_password(): + return _get_random_string(128) + + def _copy_requirements_txt_to_docker_images_dir(): copyfile(REQUIREMENTS_TXT_FILE, DOCKER_API_IMAGE_REQUIREMENTS_TXT_FILE) copyfile(REQUIREMENTS_TXT_FILE, DOCKER_WORKER_IMAGE_REQUIREMENTS_TXT_FILE) @@ -121,6 +125,7 @@ def _clean(): logger.info('🧹 Cleaning up database') subprocess.run(["docker", "volume", "rm", "okuna-api_mariadb"]) subprocess.run(["docker", "volume", "rm", "okuna-api_redisdb"]) + subprocess.run(["docker", "volume", "rm", "okuna-api_rabbitmq"]) logger.info('🧹 Cleaning up config files') _remove_file_silently(LOCAL_API_ENV_FILE) @@ -131,13 +136,13 @@ def _clean(): def _print_okuna_logo(): print(r""" - ____ _ - / __ \| | - | | | | | ___ _ _ __ __ _ + ____ _ + / __ \| | + | | | | | ___ _ _ __ __ _ | | | | |/ | | | | '_ \ / _` | | |__| | <| |_| | | | | (_| | \____/|_|\_\\__,_|_| |_|\__,_| - + """) @@ -173,6 +178,7 @@ def _ensure_has_local_api_environment_file(okuna_cli_config): "{{DJANGO_SECRET_KEY}}": okuna_cli_config['djangoSecretKey'], "{{SQL_PASSWORD}}": okuna_cli_config['sqlPassword'], "{{REDIS_PASSWORD}}": okuna_cli_config['redisPassword'], + "{{RABBITMQ_PASSWORD}}": okuna_cli_config['rabbitmqPassword'], }) @@ -190,6 +196,7 @@ def _ensure_has_docker_compose_api_environment_file(okuna_cli_config): "{{DJANGO_SECRET_KEY}}": okuna_cli_config['djangoSecretKey'], "{{SQL_PASSWORD}}": okuna_cli_config['sqlPassword'], "{{REDIS_PASSWORD}}": okuna_cli_config['redisPassword'], + "{{RABBITMQ_PASSWORD}}": okuna_cli_config['rabbitmqPassword'], }) @@ -200,10 +207,12 @@ def _ensure_has_okuna_config_file(): django_secret_key = _get_django_secret_key() mysql_password = _get_mysql_password() redis_password = _get_redis_password() + rabbitmq_password = _get_rabbitmq_password() logger.info('Generated DJANGO_SECRET_KEY=%s' % django_secret_key) logger.info('Generated SQL_PASSWORD=%s' % mysql_password) logger.info('Generated REDIS_PASSWORD=%s' % redis_password) + logger.info('Generated RABBITMQ_PASSWORD=%s' % rabbitmq_password) logger.info('Config file does not exist. Creating %s' % OKUNA_CLI_CONFIG_FILE) @@ -216,6 +225,7 @@ def _ensure_has_okuna_config_file(): "{{DJANGO_SECRET_KEY}}": django_secret_key, "{{SQL_PASSWORD}}": mysql_password, "{{REDIS_PASSWORD}}": redis_password, + "{{RABBITMQ_PASSWORD}}": rabbitmq_password, }) diff --git a/openbook/celery.py b/openbook/celery.py index e2767631..78748fbc 100644 --- a/openbook/celery.py +++ b/openbook/celery.py @@ -4,7 +4,7 @@ from celery import Celery from kombu import Queue, Exchange -from openbook.settings import CELERY_REDIS_BROKER_LOCATION, CELERY_REDIS_RESULT_BACKEND_LOCATION +from openbook.settings import CELERY_RABBITMQ_BROKER_LOCATION, CELERY_REDIS_RESULT_BACKEND_LOCATION CELERY_DEFAULT_PRIORITY_QUEUE = 'default_priority' CELERY_LOW_PRIORITY_QUEUE = 'low_priority' @@ -14,7 +14,7 @@ celery = Celery( 'openbook', - broker=CELERY_REDIS_BROKER_LOCATION, + broker=CELERY_RABBITMQ_BROKER_LOCATION, backend=CELERY_REDIS_RESULT_BACKEND_LOCATION, ) diff --git a/openbook/settings.py b/openbook/settings.py index c22a32fe..bf6db90d 100644 --- a/openbook/settings.py +++ b/openbook/settings.py @@ -156,6 +156,22 @@ JWT_ALGORITHM = os.environ.get('JWT_ALGORITHM', 'HS256') +RABBITMQ_HOST = os.environ.get('RABBITMQ_HOST', 'localhost') +RABBITMQ_PORT = int(os.environ.get('RABBITMQ_PORT', '5672')) +RABBITMQ_USERNAME = os.environ.get('RABBITMQ_USERNAME', 'guest') +RABBITMQ_PASSWORD = os.environ.get('RABBITMQ_PASSWORD', 'guest') +RABBITMQ_VHOST = os.environ.get('RABBITMQ_VHOST', '/') + +rabbitmq_vhost = '' if RABBITMQ_VHOST == '/' else '/%s' % RABBITMQ_VHOST + +RABBITMQ_LOCATION = 'amqp://%(user)s:%(password)s@%(host)s:%(port)d%(vhost)s' % { + 'user': RABBITMQ_USERNAME, + 'password': RABBITMQ_PASSWORD, + 'host': RABBITMQ_HOST, + 'port': RABBITMQ_PORT, + 'vhost': rabbitmq_vhost, +} + REDIS_HOST = os.environ.get('REDIS_HOST', 'localhost') REDIS_PORT = int(os.environ.get('REDIS_PORT', '6379')) REDIS_PASSWORD = os.environ.get('REDIS_PASSWORD') @@ -171,7 +187,8 @@ REDIS_DEFAULT_CACHE_LOCATION = '%(redis_location)s/%(db)d' % {'redis_location': REDIS_LOCATION, 'db': 0} -CELERY_REDIS_BROKER_LOCATION = '%(redis_location)s/%(db)d' % {'redis_location': REDIS_LOCATION, 'db': 1} +CELERY_RABBITMQ_BROKER_LOCATION = RABBITMQ_LOCATION +# CELERY_REDIS_BROKER_LOCATION = '%(redis_location)s/%(db)d' % {'redis_location': REDIS_LOCATION, 'db': 1} CELERY_REDIS_RESULT_BACKEND_LOCATION = '%(redis_location)s/%(db)d' % {'redis_location': REDIS_LOCATION, 'db': 2} CACHES = { diff --git a/templates/.docker-compose.env b/templates/.docker-compose.env index 65671fbe..2ea572ac 100644 --- a/templates/.docker-compose.env +++ b/templates/.docker-compose.env @@ -57,6 +57,15 @@ REDIS_PORT=6379 # [DESCRIPTION] The password for the REDIS Database. REDIS_PASSWORD={{REDIS_PASSWORD}} +# [GROUP] RabbitMQ Configuration +# [DESCRIPTION] The RabbitMQ configuration +# [REQUIRED][ALWAYS] +RABBITMQ_HOST=rabbit.okuna +RABBITMQ_PORT=5672 +RABBITMQ_USERNAME=okuna +RABBITMQ_PASSWORD={{RABBITMQ_PASSWORD}} +RABBITMQ_VHOST=okuna + # [GROUP] Top posts criteria # [DESCRIPTION] The criteria under which posts will be added to the Explore/Top posts section of the app # [OPTIONAL=2] @@ -98,4 +107,4 @@ MYSQL_DATABASE=okuna # [REQUIRED] WAIT_HOSTS:db.okuna:3306 -# ============= END DOCKER COMPOSE SPECIFIC VARIABLES ============= # \ No newline at end of file +# ============= END DOCKER COMPOSE SPECIFIC VARIABLES ============= # diff --git a/templates/.env b/templates/.env index 7e03e6b7..b814415a 100644 --- a/templates/.env +++ b/templates/.env @@ -57,6 +57,15 @@ REDIS_PORT=6380 # [DESCRIPTION] The password for the REDIS Database. If using okuna-cli, obtained from .okuna-cli.json REDIS_PASSWORD={{REDIS_PASSWORD}} +# [GROUP] RabbitMQ Configuration +# [DESCRIPTION] The RabbitMQ configuration +# [REQUIRED][ALWAYS] +RABBITMQ_HOST=127.0.0.1 +RABBITMQ_PORT=5672 +RABBITMQ_USERNAME=okuna +RABBITMQ_PASSWORD={{RABBITMQ_PASSWORD}} +RABBITMQ_VHOST=okuna + # [GROUP] Top posts criteria # [DESCRIPTION] The criteria under which posts will be added to the Explore/Top posts section of the app # [OPTIONAL=2] @@ -126,4 +135,4 @@ REDIS_PASSWORD={{REDIS_PASSWORD}} # AWS_TRANSLATE_REGION=eu-west-1 # AWS_TRANSLATE_MAX_LENGTH=10000 -# ============= END PRODUCTION VARIABLES ============= # \ No newline at end of file +# ============= END PRODUCTION VARIABLES ============= # diff --git a/templates/.okuna-cli.json b/templates/.okuna-cli.json index fbcf8f7a..add747d6 100644 --- a/templates/.okuna-cli.json +++ b/templates/.okuna-cli.json @@ -2,5 +2,6 @@ "bootstrapped": false, "djangoSecretKey": "{{DJANGO_SECRET_KEY}}", "sqlPassword": "{{SQL_PASSWORD}}", - "redisPassword": "{{REDIS_PASSWORD}}" -} \ No newline at end of file + "redisPassword": "{{REDIS_PASSWORD}}", + "rabbitmqPassword": "{{RABBITMQ_PASSWORD}}" +} From d19e6ccb157da852102f7b3aafcaab7179945945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B3zsef=20Sallai?= Date: Wed, 18 Aug 2021 14:41:07 +0000 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=90=B3=20configure=20rabbitmq=20conta?= =?UTF-8?q?iner?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .docker/rabbitmq/Dockerfile | 9 +++++++++ .docker/rabbitmq/entrypoint.sh | 11 +++++++++++ .docker/scheduler/entrypoint.sh | 2 ++ .docker/worker/entrypoint.sh | 2 ++ docker-compose-full.yml | 22 ++++++++++++++++++++++ docker-compose-services-only.yml | 20 ++++++++++++++++++++ 6 files changed, 66 insertions(+) create mode 100644 .docker/rabbitmq/Dockerfile create mode 100644 .docker/rabbitmq/entrypoint.sh diff --git a/.docker/rabbitmq/Dockerfile b/.docker/rabbitmq/Dockerfile new file mode 100644 index 00000000..8683e5cf --- /dev/null +++ b/.docker/rabbitmq/Dockerfile @@ -0,0 +1,9 @@ +FROM rabbitmq:3.9-management + +ENV RABBITMQ_PID_FILE /var/lib/rabbitmq/mnesia/rabbitmq + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +EXPOSE 15672 +ENTRYPOINT ["/entrypoint.sh"] diff --git a/.docker/rabbitmq/entrypoint.sh b/.docker/rabbitmq/entrypoint.sh new file mode 100644 index 00000000..cc68e51d --- /dev/null +++ b/.docker/rabbitmq/entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -eo pipefail + +# Create Okuna user for RabbitMQ +( rabbitmqctl wait --timeout 60 /var/lib/rabbitmq/mnesia/rabbitmq; \ +rabbitmqctl add_user $RABBITMQ_USERNAME $RABBITMQ_PASSWORD 2>/dev/null; \ +rabbitmqctl add_vhost okuna; \ +rabbitmqctl set_permissions -p okuna $RABBITMQ_USERNAME ".*" ".*" ".*"; \ +echo "Initial okuna user created successfully. You may authenticate now.") & + +rabbitmq-server $@ diff --git a/.docker/scheduler/entrypoint.sh b/.docker/scheduler/entrypoint.sh index 80b81512..fb79647f 100644 --- a/.docker/scheduler/entrypoint.sh +++ b/.docker/scheduler/entrypoint.sh @@ -5,6 +5,8 @@ cd /opt/okuna-api /wait-for-it.sh $RDS_HOSTNAME:$RDS_PORT -t 60 +/wait-for-it.sh $RABBITMQ_HOST:$RABBITMQ_PORT -t 60 + /wait-for-it.sh $REDIS_HOST:$REDIS_PORT -t 60 # install pip env deps, run migrations, collect media, start the server diff --git a/.docker/worker/entrypoint.sh b/.docker/worker/entrypoint.sh index 80b81512..fb79647f 100644 --- a/.docker/worker/entrypoint.sh +++ b/.docker/worker/entrypoint.sh @@ -5,6 +5,8 @@ cd /opt/okuna-api /wait-for-it.sh $RDS_HOSTNAME:$RDS_PORT -t 60 +/wait-for-it.sh $RABBITMQ_HOST:$RABBITMQ_PORT -t 60 + /wait-for-it.sh $REDIS_HOST:$REDIS_PORT -t 60 # install pip env deps, run migrations, collect media, start the server diff --git a/docker-compose-full.yml b/docker-compose-full.yml index 226b60a8..05e90f4d 100644 --- a/docker-compose-full.yml +++ b/docker-compose-full.yml @@ -10,6 +10,7 @@ services: extra_hosts: - db.okuna:172.16.16.4 - redis.okuna:172.16.16.5 + - rabbit.okuna:172.16.16.6 volumes: - ./:/opt/okuna-api - ./.docker-cache/pip:/root/.cache/pip @@ -22,6 +23,7 @@ services: depends_on: - db - redis + - rabbit env_file: - .docker-compose.env worker: @@ -33,6 +35,7 @@ services: extra_hosts: - db.okuna:172.16.16.4 - redis.okuna:172.16.16.5 + - rabbit.okuna:172.16.16.6 volumes: - ./:/opt/okuna-api - ./.docker-cache/pip:/root/.cache/pip @@ -53,6 +56,7 @@ services: extra_hosts: - db.okuna:172.16.16.4 - redis.okuna:172.16.16.5 + - rabbit.okuna:172.16.16.6 volumes: - ./:/opt/okuna-api - ./.docker-cache/pip:/root/.cache/pip @@ -90,10 +94,28 @@ services: - .docker-compose.env volumes: - redisdb:/bitnami/redis/data + rabbit: + container_name: rabbit + build: + dockerfile: Dockerfile + context: ./.docker/rabbitmq + privileged: true + ports: + - 5672 + - 15672 + volumes: + - rabbitmq:/var/lib/rabbitmq + networks: + okuna: + ipv4_address: 172.16.16.6 + env_file: + - .docker-compose.env + volumes: mariadb: redisdb: + rabbitmq: networks: okuna: diff --git a/docker-compose-services-only.yml b/docker-compose-services-only.yml index b492a32b..4905d683 100644 --- a/docker-compose-services-only.yml +++ b/docker-compose-services-only.yml @@ -10,6 +10,7 @@ services: extra_hosts: - db.okuna:172.16.16.4 - redis.okuna:172.16.16.5 + - rabbit.okuna:172.16.16.6 volumes: - ./:/opt/okuna-api - ./.docker-cache/pip:/root/.cache/pip @@ -20,6 +21,7 @@ services: depends_on: - db - redis + - rabbit env_file: - .docker-compose.env scheduler: @@ -31,6 +33,7 @@ services: extra_hosts: - db.okuna:172.16.16.4 - redis.okuna:172.16.16.5 + - rabbit.okuna:172.16.16.6 volumes: - ./:/opt/okuna-api - ./.docker-cache/pip:/root/.cache/pip @@ -68,10 +71,27 @@ services: - .docker-compose.env volumes: - redisdb:/bitnami/redis/data + rabbit: + container_name: rabbit + build: + dockerfile: Dockerfile + context: ./.docker/rabbitmq + privileged: true + ports: + - 5672 + - 15672 + volumes: + - rabbitmq:/var/lib/rabbitmq + networks: + okuna: + ipv4_address: 172.16.16.6 + env_file: + - .docker-compose.env volumes: mariadb: redisdb: + rabbitmq: networks: okuna: From 8c7272fb1b802b8afa2c9488ed6bc0e2a5e8d355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B3zsef=20Sallai?= Date: Mon, 23 Aug 2021 20:01:56 +0300 Subject: [PATCH 5/5] =?UTF-8?q?=E2=9C=A8=20add=20autoretries=20to=20celery?= =?UTF-8?q?=20tasks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- openbook_notifications/jobs.py | 2 +- openbook_posts/jobs.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/openbook_notifications/jobs.py b/openbook_notifications/jobs.py index b0338ee4..aa4252ac 100644 --- a/openbook_notifications/jobs.py +++ b/openbook_notifications/jobs.py @@ -10,7 +10,7 @@ app_auth_key=settings.ONE_SIGNAL_API_KEY ) -@celery.task(queue='default_priority') +@celery.task(queue='default_priority', autoretry_for=(Exception,)) def send_notification_to_user_with_id(user_id, notification): User = get_user_model() user = User.objects.only('username', 'uuid', 'id').get(pk=user_id) diff --git a/openbook_posts/jobs.py b/openbook_posts/jobs.py index 9726fb7f..a65afc04 100644 --- a/openbook_posts/jobs.py +++ b/openbook_posts/jobs.py @@ -12,7 +12,7 @@ logger = logging.getLogger(__name__) -@celery.task(queue='low_priority') +@celery.task(queue='low_priority', autoretry_for=(Exception,)) def flush_draft_posts(): """ This job should be scheduled to get all pending draft posts for a day and remove them @@ -31,7 +31,7 @@ def flush_draft_posts(): return 'Flushed %s posts' % str(flushed_posts) -@celery.task(queue='high_priority') +@celery.task(queue='high_priority', autoretry_for=(Exception,)) def process_post_media(post_id): """ This job is called to process post media and mark it as published @@ -52,7 +52,7 @@ def process_post_media(post_id): logger.info('Processed media of post with id: %d' % post_id) -@celery.task(queue='low_priority') +@celery.task(queue='low_priority', autoretry_for=(Exception,)) def curate_top_posts(): """ Curates the top posts. @@ -118,7 +118,7 @@ def curate_top_posts(): return 'Checked: %d. Curated: %d' % (total_checked_posts, total_curated_posts) -@celery.task(queue='low_priority') +@celery.task(queue='low_priority', autoretry_for=(Exception,)) def clean_top_posts(): """ Cleans up top posts, that no longer meet the criteria. @@ -197,7 +197,7 @@ def _add_post_to_top_post(post): return None -@celery.task(queue='low_priority') +@celery.task(queue='low_priority', autoretry_for=(Exception,)) def curate_trending_posts(): """ Curates the trending posts. @@ -248,7 +248,7 @@ def curate_trending_posts(): return 'Curated: %d posts' % posts.count() -@celery.task(queue='low_priority') +@celery.task(queue='low_priority', autoretry_for=(Exception,)) def bootstrap_trending_posts(): """ Bootstraps the trending posts. @@ -302,7 +302,7 @@ def bootstrap_trending_posts(): return 'Checked: %d. Curated: %d' % (total_checked_posts, total_curated_posts) -@celery.task(queue='low_priority') +@celery.task(queue='low_priority', autoretry_for=(Exception,)) def clean_trending_posts(): """ Cleans trending posts.