diff --git a/README.md b/README.md index f861802..5f973ba 100644 --- a/README.md +++ b/README.md @@ -65,16 +65,18 @@ systemctl restart uwsgi systemctl reload nginx ``` -## CRON - -Mostly for cron use, there is a `manage.sh` script that prepares -the environment and calls `manage.py` through poetry. - -```cron -# Send account expiration notice e-mails -0 */6 * * * /home/ccvpn/ccvpn3/manage.sh expire_notify -# Expire old and cancelled payments -0 0 * * * /home/ccvpn/ccvpn3/manage.sh expire_payments -# [bitcoin] Check for incoming payments -*/5 * * * * /home/ccvpn/ccvpn3/manage.sh check_btc_payments +## Celery + +``` +# run celery background processes +poetry run celery -A ccvpn worker -l info +poetry run celery -A ccvpn beat -l info +# celery flower monitoring web ui +poetry run flower --port=5555 -A ccvpn + +# ~ with systemd services ~ + +cp ./deploy/ccvpn-*.service /etc/systemd/ +systemctl daemon-reload +systemctl enable --now ccvpn-{worker,scheduler,flower}.service ``` diff --git a/deploy/ccvpn-flower.service b/deploy/ccvpn-flower.service new file mode 100644 index 0000000..ec81ac5 --- /dev/null +++ b/deploy/ccvpn-flower.service @@ -0,0 +1,13 @@ +[Unit] +Description=ccvpn worker +After=network.target + +[Service] +Type=simple +ExecStart=/home/ccvpn/ccvpn3/.venv/bin/celery flower -A ccvpn -l INFO +WorkingDirectory=/home/ccvpn/ccvpn3/ +User=ccvpn +Group=ccvpn + +[Install] +WantedBy=multi-user.target diff --git a/deploy/ccvpn-scheduler.service b/deploy/ccvpn-scheduler.service new file mode 100644 index 0000000..45255be --- /dev/null +++ b/deploy/ccvpn-scheduler.service @@ -0,0 +1,13 @@ +[Unit] +Description=ccvpn scheduler +After=network.target + +[Service] +Type=simple +ExecStart=/home/ccvpn/ccvpn3/.venv/bin/celery beat -A ccvpn -l INFO +WorkingDirectory=/home/ccvpn/ccvpn3/ +User=ccvpn +Group=ccvpn + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/deploy/ccvpn-worker.service b/deploy/ccvpn-worker.service new file mode 100644 index 0000000..e4423b0 --- /dev/null +++ b/deploy/ccvpn-worker.service @@ -0,0 +1,13 @@ +[Unit] +Description=ccvpn worker +After=network.target + +[Service] +Type=simple +ExecStart=/home/ccvpn/ccvpn3/.venv/bin/celery worker -A ccvpn -l INFO +WorkingDirectory=/home/ccvpn/ccvpn3/ +User=ccvpn +Group=ccvpn + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index d05eaa3..1e92e85 100644 --- a/poetry.lock +++ b/poetry.lock @@ -9,6 +9,14 @@ version = "2.6.1" [package.dependencies] vine = ">=1.1.3,<5.0.0a1" +[[package]] +category = "dev" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +name = "appdirs" +optional = false +python-versions = "*" +version = "1.4.4" + [[package]] category = "main" description = "ASGI specs, helper code, and adapters" @@ -45,6 +53,32 @@ optional = false python-versions = "*" version = "3.6.3.0" +[[package]] +category = "dev" +description = "The uncompromising code formatter." +name = "black" +optional = false +python-versions = ">=3.6" +version = "20.8b1" + +[package.dependencies] +appdirs = "*" +click = ">=7.1.2" +mypy-extensions = ">=0.4.3" +pathspec = ">=0.6,<1" +regex = ">=2020.1.8" +toml = ">=0.10.1" +typed-ast = ">=1.4.0" +typing-extensions = ">=3.7.4" + +[package.dependencies.dataclasses] +python = "<3.7" +version = ">=0.6" + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] + [[package]] category = "main" description = "Distributed Task Queue." @@ -109,6 +143,14 @@ optional = false python-versions = "*" version = "3.0.4" +[[package]] +category = "dev" +description = "Composable command line interface toolkit" +name = "click" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "7.1.2" + [[package]] category = "dev" description = "Cross-platform colored terminal text." @@ -118,6 +160,15 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" version = "0.4.3" +[[package]] +category = "dev" +description = "A backport of the dataclasses module for Python 3.6" +marker = "python_version < \"3.7\"" +name = "dataclasses" +optional = false +python-versions = ">=3.6, <3.7" +version = "0.7" + [[package]] category = "main" description = "A high-level Python Web framework that encourages rapid development and clean, pragmatic design." @@ -226,6 +277,7 @@ lcoreapi = "*" reference = "1bdf87ac95a4c7c8c95c8704d302e2e2e654dfea" type = "git" url = "https://git.packetimpact.net/lvpn/django-lcore.git" + [[package]] category = "main" description = "Pickled object field for Django" @@ -398,6 +450,7 @@ requests = "*" reference = "df771a7dbd1a2166a9a873539f976865f6f6a630" type = "git" url = "https://git.packetimpact.net/lvpn/lcoreapi.git" + [[package]] category = "main" description = "Python implementation of Markdown." @@ -422,6 +475,22 @@ optional = false python-versions = "*" version = "0.6.1" +[[package]] +category = "dev" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +name = "mypy-extensions" +optional = false +python-versions = "*" +version = "0.4.3" + +[[package]] +category = "dev" +description = "Utility library for gitignore style pattern matching of file paths." +name = "pathspec" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.8.0" + [[package]] category = "main" description = "Python client for the Prometheus monitoring system." @@ -578,6 +647,14 @@ version = "3.5.3" [package.extras] hiredis = ["hiredis (>=0.1.3)"] +[[package]] +category = "dev" +description = "Alternative regular expression module, to replace re." +name = "regex" +optional = false +python-versions = "*" +version = "2020.7.14" + [[package]] category = "main" description = "Python HTTP for Humans." @@ -645,12 +722,19 @@ version = "6.0.4" [[package]] category = "dev" description = "a fork of Python 2 and 3 ast modules with type comment support" -marker = "implementation_name == \"cpython\" and python_version < \"3.8\"" name = "typed-ast" optional = false python-versions = "*" version = "1.4.1" +[[package]] +category = "dev" +description = "Backported and Experimental Type Hints for Python 3.5+" +name = "typing-extensions" +optional = false +python-versions = "*" +version = "3.7.4.3" + [[package]] category = "main" description = "HTTP library with thread-safe connection pooling, file post, and more." @@ -694,7 +778,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [metadata] -content-hash = "813313ef4a9d1ddbb44d79850bea6ee0f819458d6a69f34babd9abd1490fb31a" +content-hash = "87c64dd6869c8f77b6180664b009d33f6de6808036f0d3668add3b3ea2b8bf52" python-versions = "^3.6" [metadata.files] @@ -702,6 +786,10 @@ amqp = [ {file = "amqp-2.6.1-py2.py3-none-any.whl", hash = "sha256:aa7f313fb887c91f15474c1229907a04dac0b8135822d6603437803424c0aa59"}, {file = "amqp-2.6.1.tar.gz", hash = "sha256:70cdb10628468ff14e57ec2f751c7aa9e48e7e3651cfd62d431213c0c4e58f21"}, ] +appdirs = [ + {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, + {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, +] asgiref = [ {file = "asgiref-3.2.10-py3-none-any.whl", hash = "sha256:9fc6fb5d39b8af147ba40765234fa822b39818b12cc80b35ad9b0cef3a476aed"}, {file = "asgiref-3.2.10.tar.gz", hash = "sha256:7e51911ee147dd685c3c8b805c0ad0cb58d360987b56953878f8c06d2d1c6f1a"}, @@ -714,6 +802,10 @@ billiard = [ {file = "billiard-3.6.3.0-py3-none-any.whl", hash = "sha256:bff575450859a6e0fbc2f9877d9b715b0bbc07c3565bb7ed2280526a0cdf5ede"}, {file = "billiard-3.6.3.0.tar.gz", hash = "sha256:d91725ce6425f33a97dfa72fb6bfef0e47d4652acd98a032bd1a7fbf06d5fa6a"}, ] +black = [ + {file = "black-20.8b1-py3-none-any.whl", hash = "sha256:70b62ef1527c950db59062cda342ea224d772abdf6adc58b86a45421bab20a6b"}, + {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, +] celery = [ {file = "celery-4.4.7-py2.py3-none-any.whl", hash = "sha256:a92e1d56e650781fb747032a3997d16236d037c8199eacd5217d1a72893bca45"}, {file = "celery-4.4.7.tar.gz", hash = "sha256:d220b13a8ed57c78149acf82c006785356071844afe0b27012a4991d44026f9f"}, @@ -726,10 +818,18 @@ chardet = [ {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, ] +click = [ + {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, + {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, +] colorama = [ {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"}, ] +dataclasses = [ + {file = "dataclasses-0.7-py3-none-any.whl", hash = "sha256:3459118f7ede7c8bea0fe795bff7c6c2ce287d01dd226202f7c9ebc0610a7836"}, + {file = "dataclasses-0.7.tar.gz", hash = "sha256:494a6dcae3b8bcf80848eea2ef64c0cc5cd307ffc263e17cdf42f3e5420808e6"}, +] django = [ {file = "Django-3.1-py3-none-any.whl", hash = "sha256:1a63f5bb6ff4d7c42f62a519edc2adbb37f9b78068a5a862beff858b68e3dc8b"}, {file = "Django-3.1.tar.gz", hash = "sha256:2d390268a13c655c97e0e2ede9d117007996db692c1bb93eabebd4fb7ea7012b"}, @@ -830,6 +930,14 @@ mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] +mypy-extensions = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] +pathspec = [ + {file = "pathspec-0.8.0-py2.py3-none-any.whl", hash = "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0"}, + {file = "pathspec-0.8.0.tar.gz", hash = "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061"}, +] prometheus-client = [ {file = "prometheus_client-0.8.0-py2.py3-none-any.whl", hash = "sha256:983c7ac4b47478720db338f1491ef67a100b474e3bc7dafcbaefb7d0b8f9b01c"}, {file = "prometheus_client-0.8.0.tar.gz", hash = "sha256:c6e6b706833a6bd1fd51711299edee907857be10ece535126a158f911ee80915"}, @@ -922,6 +1030,29 @@ redis = [ {file = "redis-3.5.3-py2.py3-none-any.whl", hash = "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24"}, {file = "redis-3.5.3.tar.gz", hash = "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2"}, ] +regex = [ + {file = "regex-2020.7.14-cp27-cp27m-win32.whl", hash = "sha256:e46d13f38cfcbb79bfdb2964b0fe12561fe633caf964a77a5f8d4e45fe5d2ef7"}, + {file = "regex-2020.7.14-cp27-cp27m-win_amd64.whl", hash = "sha256:6961548bba529cac7c07af2fd4d527c5b91bb8fe18995fed6044ac22b3d14644"}, + {file = "regex-2020.7.14-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c50a724d136ec10d920661f1442e4a8b010a4fe5aebd65e0c2241ea41dbe93dc"}, + {file = "regex-2020.7.14-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:8a51f2c6d1f884e98846a0a9021ff6861bdb98457879f412fdc2b42d14494067"}, + {file = "regex-2020.7.14-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:9c568495e35599625f7b999774e29e8d6b01a6fb684d77dee1f56d41b11b40cd"}, + {file = "regex-2020.7.14-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:51178c738d559a2d1071ce0b0f56e57eb315bcf8f7d4cf127674b533e3101f88"}, + {file = "regex-2020.7.14-cp36-cp36m-win32.whl", hash = "sha256:9eddaafb3c48e0900690c1727fba226c4804b8e6127ea409689c3bb492d06de4"}, + {file = "regex-2020.7.14-cp36-cp36m-win_amd64.whl", hash = "sha256:14a53646369157baa0499513f96091eb70382eb50b2c82393d17d7ec81b7b85f"}, + {file = "regex-2020.7.14-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:1269fef3167bb52631ad4fa7dd27bf635d5a0790b8e6222065d42e91bede4162"}, + {file = "regex-2020.7.14-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:d0a5095d52b90ff38592bbdc2644f17c6d495762edf47d876049cfd2968fbccf"}, + {file = "regex-2020.7.14-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:4c037fd14c5f4e308b8370b447b469ca10e69427966527edcab07f52d88388f7"}, + {file = "regex-2020.7.14-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:bc3d98f621898b4a9bc7fecc00513eec8f40b5b83913d74ccb445f037d58cd89"}, + {file = "regex-2020.7.14-cp37-cp37m-win32.whl", hash = "sha256:46bac5ca10fb748d6c55843a931855e2727a7a22584f302dd9bb1506e69f83f6"}, + {file = "regex-2020.7.14-cp37-cp37m-win_amd64.whl", hash = "sha256:0dc64ee3f33cd7899f79a8d788abfbec168410be356ed9bd30bbd3f0a23a7204"}, + {file = "regex-2020.7.14-cp38-cp38-manylinux1_i686.whl", hash = "sha256:5ea81ea3dbd6767873c611687141ec7b06ed8bab43f68fad5b7be184a920dc99"}, + {file = "regex-2020.7.14-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:bbb332d45b32df41200380fff14712cb6093b61bd142272a10b16778c418e98e"}, + {file = "regex-2020.7.14-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:c11d6033115dc4887c456565303f540c44197f4fc1a2bfb192224a301534888e"}, + {file = "regex-2020.7.14-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:75aaa27aa521a182824d89e5ab0a1d16ca207318a6b65042b046053cfc8ed07a"}, + {file = "regex-2020.7.14-cp38-cp38-win32.whl", hash = "sha256:d6cff2276e502b86a25fd10c2a96973fdb45c7a977dca2138d661417f3728341"}, + {file = "regex-2020.7.14-cp38-cp38-win_amd64.whl", hash = "sha256:7a2dd66d2d4df34fa82c9dc85657c5e019b87932019947faece7983f2089a840"}, + {file = "regex-2020.7.14.tar.gz", hash = "sha256:3a3af27a8d23143c49a3420efe5b3f8cf1a48c6fc8bc6856b03f638abc1833bb"}, +] requests = [ {file = "requests-2.24.0-py2.py3-none-any.whl", hash = "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"}, {file = "requests-2.24.0.tar.gz", hash = "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b"}, @@ -976,6 +1107,11 @@ typed-ast = [ {file = "typed_ast-1.4.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34"}, {file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"}, ] +typing-extensions = [ + {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, + {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, + {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"}, +] urllib3 = [ {file = "urllib3-1.25.10-py2.py3-none-any.whl", hash = "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"}, {file = "urllib3-1.25.10.tar.gz", hash = "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a"}, diff --git a/pyproject.toml b/pyproject.toml index ef63ae8..baac3b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ flower = "^0.9.5" [tool.poetry.dev-dependencies] pylint = "^2.3" pylint-django = "^2.0.14" +black = {version = "^20.8b1", allow-prereleases = true} [build-system] requires = ["poetry>=0.12"]