Add basic `async` support

2023-08-07 17:31:14 -04:00 · 2023-08-07 17:31:14 -04:00 · fc08c858e1
parent ce5c516de1
commit fc08c858e1
5 changed files with 158 additions and 47 deletions
--- a/poetry.lock
+++ b/poetry.lock
@ -1,5 +1,26 @@
 # This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand.

+[[package]]
+name = "anyio"
+version = "3.7.1"
+description = "High level compatibility layer for multiple asynchronous event loop implementations"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"},
+    {file = "anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780"},
+]
+
+[package.dependencies]
+exceptiongroup = {version = "*", markers = "python_version < \"3.11\""}
+idna = ">=2.8"
+sniffio = ">=1.1"
+
+[package.extras]
+doc = ["Sphinx", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-jquery"]
+test = ["anyio[trio]", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"]
+trio = ["trio (<0.22)"]
+
 [[package]]
 name = "black"
 version = "23.7.0"
@ -165,6 +186,75 @@ files = [
    {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
 ]

+[[package]]
+name = "exceptiongroup"
+version = "1.1.2"
+description = "Backport of PEP 654 (exception groups)"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"},
+    {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"},
+]
+
+[package.extras]
+test = ["pytest (>=6)"]
+
+[[package]]
+name = "h11"
+version = "0.14.0"
+description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"},
+    {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"},
+]
+
+[[package]]
+name = "httpcore"
+version = "0.17.3"
+description = "A minimal low-level HTTP client."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "httpcore-0.17.3-py3-none-any.whl", hash = "sha256:c2789b767ddddfa2a5782e3199b2b7f6894540b17b16ec26b2c4d8e103510b87"},
+    {file = "httpcore-0.17.3.tar.gz", hash = "sha256:a6f30213335e34c1ade7be6ec7c47f19f50c56db36abef1a9dfa3815b1cb3888"},
+]
+
+[package.dependencies]
+anyio = ">=3.0,<5.0"
+certifi = "*"
+h11 = ">=0.13,<0.15"
+sniffio = "==1.*"
+
+[package.extras]
+http2 = ["h2 (>=3,<5)"]
+socks = ["socksio (==1.*)"]
+
+[[package]]
+name = "httpx"
+version = "0.24.1"
+description = "The next generation HTTP client."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "httpx-0.24.1-py3-none-any.whl", hash = "sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd"},
+    {file = "httpx-0.24.1.tar.gz", hash = "sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd"},
+]
+
+[package.dependencies]
+certifi = "*"
+httpcore = ">=0.15.0,<0.18.0"
+idna = "*"
+sniffio = "*"
+
+[package.extras]
+brotli = ["brotli", "brotlicffi"]
+cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
+http2 = ["h2 (>=3,<5)"]
+socks = ["socksio (==1.*)"]
+
 [[package]]
 name = "idna"
 version = "3.4"
@ -303,6 +393,17 @@ files = [
    {file = "ruff-0.0.281.tar.gz", hash = "sha256:bab2cdfa78754315cccc2b4d46ad6181aabb29e89747a3b135a4b85e11baa025"},
 ]

+[[package]]
+name = "sniffio"
+version = "1.3.0"
+description = "Sniff out which async library your code is running under"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"},
+    {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"},
+]
+
 [[package]]
 name = "tomli"
 version = "2.0.1"
@ -348,4 +449,4 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "d1e33ec5def7b19f6fd09ac80725a9a9f24da3274b075ee37eb53fe2b2f54219"
+content-hash = "0c9985ad60dc4fee35917dd0ff673af48019ca19a009b5185f6eff9343428a84"
--- a/pyproject.toml
+++ b/pyproject.toml
@ -18,6 +18,7 @@ python = "^3.10"
 requests = "^2.31.0"
 loguru = "^0.7.0"
 python-dotenv = "^1.0.0"
+httpx = "^0.24.1"


 [tool.poetry.group.dev.dependencies]
--- a/src/main.py
+++ b/src/main.py
@ -5,7 +5,7 @@ import argparse
 import os
 from pathlib import Path
 from sys import exit, stderr
-
+import asyncio
 import dotenv
 from loguru import logger

@ -67,7 +67,7 @@ def main():
        "--whitelists",
        "-w",
        help="Either a whitelist hosts file or a directory containing whitelist hosts files",  # noqa E501
-        default="whitelist.txt",  # Not really needed because the apply_whitelists function will default to this  # noqa: E501
+        default="whitelists",  # Not really needed because the apply_whitelists function will default to this  # noqa: E501
    )
    # Add subcommand: delete
    delete_parser = subparsers.add_parser(
@ -108,7 +108,8 @@ def main():

    try:
        args.func(args)
-    except AttributeError:
+    except AttributeError as e:
+        logger.debug(e)
        logger.error("No subcommand specified")
        argparser.print_help()
        exit(1)
@ -119,7 +120,7 @@ def upload_to_cloudflare(args):
    blocklists = upload.get_blocklists(args.blocklists)
    blocklists = upload.apply_whitelists(blocklists, args.whitelists)
    lists = upload.split_list(blocklists)
-    upload.upload_to_cloudflare(lists, ACCOUNT_ID, TOKEN)
+    asyncio.run(upload.upload_to_cloudflare(lists, ACCOUNT_ID, TOKEN))
    cloud_lists = utils.get_lists(ACCOUNT_ID, TOKEN)
    cloud_lists = utils.filter_adblock_lists(cloud_lists)
    upload.create_dns_policy(cloud_lists, ACCOUNT_ID, TOKEN)
@ -131,7 +132,7 @@ def delete_from_cloudflare(args):
    delete.delete_adblock_policy(rules, ACCOUNT_ID, TOKEN)
    lists = utils.get_lists(ACCOUNT_ID, TOKEN)
    lists = utils.filter_adblock_lists(lists)
-    delete.delete_adblock_list(lists, ACCOUNT_ID, TOKEN)
+    asyncio.run(delete.delete_adblock_list(lists, ACCOUNT_ID, TOKEN))

 def set_primary_logger(log_level):
    logger.remove()
--- a/src/utils/delete.py
+++ b/src/utils/delete.py
@ -1,19 +1,22 @@
 # This is a scriprt to undo the changes made by adblock-zerotrust.py

 import requests
-
+import httpx 
+import asyncio
 from . import utils


-def delete_adblock_list(lists: dict, account_id: str, token: str):
+async def delete_adblock_list(lists: dict, account_id: str, token: str):
    try:
+        async with httpx.AsyncClient() as client:
            for lst in lists:
                url = f'https://api.cloudflare.com/client/v4/accounts/{account_id}/gateway/lists/{lst["id"]}'
                headers = {
                    "Authorization": f"Bearer {token}",
                    "Content-Type": "application/json",
                }
-            response = requests.delete(url, headers=headers, timeout=10)
+                # response = requests.delete(url, headers=headers, timeout=10)
+                response = await client.delete(url, headers=headers, timeout=10)
                if response.status_code != 200:
                    print(f"Error deleting list: {response.text}")
                else:
@ -48,7 +51,7 @@ def main():
    token = input("Enter your Cloudflare API token: ")

    rules = utils.get_gateway_rules(account_id, token)
-    delete_adblock_policy(rules, account_id, token)
+    asyncio.run(utils.delete_adblock_list(rules, account_id, token))
    lists = utils.get_lists(account_id, token)
    lists = utils.filter_adblock_lists(lists)
    delete_adblock_list(lists, account_id, token)
--- a/src/utils/upload.py
+++ b/src/utils/upload.py
@ -1,6 +1,8 @@
 import pathlib

 import requests
+import asyncio
+import httpx

 from . import utils

@ -46,7 +48,8 @@ def split_list(blocklists):
    return lists


-def upload_to_cloudflare(lists, account_id: str, token: str) -> None:
+async def upload_to_cloudflare(lists, account_id: str, token: str) -> None: 
+    async with httpx.AsyncClient() as client:
        for i, lst in enumerate(lists):
            list_name = f"adblock-list-{i + 1}"
            url = (
@ -69,10 +72,12 @@ def upload_to_cloudflare(lists, account_id: str, token: str) -> None:
                    for x in lst
                ],
            }
-        response = requests.post(url, headers=headers, json=data, timeout=10)
+            response = await client.post(url, headers=headers, json=data)
            print(f"Uploaded {list_name} to Cloudflare")
            if response.status_code != 200:
                print(f"Error uploading {list_name}: {response.text}")
+                exit(1)
+


 def create_dns_policy(lists, account_id: str, token: str) -> None:
@ -108,7 +113,7 @@ def main():
    blocklists = get_blocklists()
    blocklists = apply_whitelists(blocklists)
    lists = split_list(blocklists)
-    upload_to_cloudflare(lists, account_id, token)
+    asyncio.run(upload_to_cloudflare(lists, account_id, token))
    cloud_lists = utils.get_lists(account_id, token)
    cloud_lists = utils.filter_adblock_lists(cloud_lists)
    create_dns_policy(cloud_lists, account_id, token)