From fc08c858e1b61643205774daf7ff49570977e6ee Mon Sep 17 00:00:00 2001 From: slashtechno <77907286+slashtechno@users.noreply.github.com> Date: Mon, 7 Aug 2023 17:31:14 -0400 Subject: [PATCH] Add basic `async` support --- poetry.lock | 103 +++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + src/__main__.py | 11 ++--- src/utils/delete.py | 31 +++++++------ src/utils/upload.py | 59 +++++++++++++------------ 5 files changed, 158 insertions(+), 47 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0a165e4..53a1776 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,26 @@ # This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +[[package]] +name = "anyio" +version = "3.7.1" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.7" +files = [ + {file = "anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"}, + {file = "anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780"}, +] + +[package.dependencies] +exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" + +[package.extras] +doc = ["Sphinx", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-jquery"] +test = ["anyio[trio]", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (<0.22)"] + [[package]] name = "black" version = "23.7.0" @@ -165,6 +186,75 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "exceptiongroup" +version = "1.1.2" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, + {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "httpcore" +version = "0.17.3" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.7" +files = [ + {file = "httpcore-0.17.3-py3-none-any.whl", hash = "sha256:c2789b767ddddfa2a5782e3199b2b7f6894540b17b16ec26b2c4d8e103510b87"}, + {file = "httpcore-0.17.3.tar.gz", hash = "sha256:a6f30213335e34c1ade7be6ec7c47f19f50c56db36abef1a9dfa3815b1cb3888"}, +] + +[package.dependencies] +anyio = ">=3.0,<5.0" +certifi = "*" +h11 = ">=0.13,<0.15" +sniffio = "==1.*" + +[package.extras] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] + +[[package]] +name = "httpx" +version = "0.24.1" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.7" +files = [ + {file = "httpx-0.24.1-py3-none-any.whl", hash = "sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd"}, + {file = "httpx-0.24.1.tar.gz", hash = "sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd"}, +] + +[package.dependencies] +certifi = "*" +httpcore = ">=0.15.0,<0.18.0" +idna = "*" +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] + [[package]] name = "idna" version = "3.4" @@ -303,6 +393,17 @@ files = [ {file = "ruff-0.0.281.tar.gz", hash = "sha256:bab2cdfa78754315cccc2b4d46ad6181aabb29e89747a3b135a4b85e11baa025"}, ] +[[package]] +name = "sniffio" +version = "1.3.0" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, + {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, +] + [[package]] name = "tomli" version = "2.0.1" @@ -348,4 +449,4 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "d1e33ec5def7b19f6fd09ac80725a9a9f24da3274b075ee37eb53fe2b2f54219" +content-hash = "0c9985ad60dc4fee35917dd0ff673af48019ca19a009b5185f6eff9343428a84" diff --git a/pyproject.toml b/pyproject.toml index c40a614..4537618 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ python = "^3.10" requests = "^2.31.0" loguru = "^0.7.0" python-dotenv = "^1.0.0" +httpx = "^0.24.1" [tool.poetry.group.dev.dependencies] diff --git a/src/__main__.py b/src/__main__.py index d2660f1..79ab601 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -5,7 +5,7 @@ import argparse import os from pathlib import Path from sys import exit, stderr - +import asyncio import dotenv from loguru import logger @@ -67,7 +67,7 @@ def main(): "--whitelists", "-w", help="Either a whitelist hosts file or a directory containing whitelist hosts files", # noqa E501 - default="whitelist.txt", # Not really needed because the apply_whitelists function will default to this # noqa: E501 + default="whitelists", # Not really needed because the apply_whitelists function will default to this # noqa: E501 ) # Add subcommand: delete delete_parser = subparsers.add_parser( @@ -108,7 +108,8 @@ def main(): try: args.func(args) - except AttributeError: + except AttributeError as e: + logger.debug(e) logger.error("No subcommand specified") argparser.print_help() exit(1) @@ -119,7 +120,7 @@ def upload_to_cloudflare(args): blocklists = upload.get_blocklists(args.blocklists) blocklists = upload.apply_whitelists(blocklists, args.whitelists) lists = upload.split_list(blocklists) - upload.upload_to_cloudflare(lists, ACCOUNT_ID, TOKEN) + asyncio.run(upload.upload_to_cloudflare(lists, ACCOUNT_ID, TOKEN)) cloud_lists = utils.get_lists(ACCOUNT_ID, TOKEN) cloud_lists = utils.filter_adblock_lists(cloud_lists) upload.create_dns_policy(cloud_lists, ACCOUNT_ID, TOKEN) @@ -131,7 +132,7 @@ def delete_from_cloudflare(args): delete.delete_adblock_policy(rules, ACCOUNT_ID, TOKEN) lists = utils.get_lists(ACCOUNT_ID, TOKEN) lists = utils.filter_adblock_lists(lists) - delete.delete_adblock_list(lists, ACCOUNT_ID, TOKEN) + asyncio.run(delete.delete_adblock_list(lists, ACCOUNT_ID, TOKEN)) def set_primary_logger(log_level): logger.remove() diff --git a/src/utils/delete.py b/src/utils/delete.py index 878e3c2..9dd7825 100644 --- a/src/utils/delete.py +++ b/src/utils/delete.py @@ -1,23 +1,26 @@ # This is a scriprt to undo the changes made by adblock-zerotrust.py import requests - +import httpx +import asyncio from . import utils -def delete_adblock_list(lists: dict, account_id: str, token: str): +async def delete_adblock_list(lists: dict, account_id: str, token: str): try: - for lst in lists: - url = f'https://api.cloudflare.com/client/v4/accounts/{account_id}/gateway/lists/{lst["id"]}' - headers = { - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - } - response = requests.delete(url, headers=headers, timeout=10) - if response.status_code != 200: - print(f"Error deleting list: {response.text}") - else: - print(f'Deleted list {lst["name"]}') + async with httpx.AsyncClient() as client: + for lst in lists: + url = f'https://api.cloudflare.com/client/v4/accounts/{account_id}/gateway/lists/{lst["id"]}' + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + # response = requests.delete(url, headers=headers, timeout=10) + response = await client.delete(url, headers=headers, timeout=10) + if response.status_code != 200: + print(f"Error deleting list: {response.text}") + else: + print(f'Deleted list {lst["name"]}') except TypeError as e: if str(e) == "'NoneType' object is not iterable": print("No lists found") @@ -48,7 +51,7 @@ def main(): token = input("Enter your Cloudflare API token: ") rules = utils.get_gateway_rules(account_id, token) - delete_adblock_policy(rules, account_id, token) + asyncio.run(utils.delete_adblock_list(rules, account_id, token)) lists = utils.get_lists(account_id, token) lists = utils.filter_adblock_lists(lists) delete_adblock_list(lists, account_id, token) diff --git a/src/utils/upload.py b/src/utils/upload.py index b394152..1f4c092 100644 --- a/src/utils/upload.py +++ b/src/utils/upload.py @@ -1,6 +1,8 @@ import pathlib import requests +import asyncio +import httpx from . import utils @@ -46,33 +48,36 @@ def split_list(blocklists): return lists -def upload_to_cloudflare(lists, account_id: str, token: str) -> None: - for i, lst in enumerate(lists): - list_name = f"adblock-list-{i + 1}" - url = ( - f"https://api.cloudflare.com/client/v4/accounts/{account_id}/gateway/lists" - ) - headers = { - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - } +async def upload_to_cloudflare(lists, account_id: str, token: str) -> None: + async with httpx.AsyncClient() as client: + for i, lst in enumerate(lists): + list_name = f"adblock-list-{i + 1}" + url = ( + f"https://api.cloudflare.com/client/v4/accounts/{account_id}/gateway/lists" + ) + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + + data = { + "name": list_name, + "type": "DOMAIN", + "description": "A blocklist of ad domains", + # Writing this program, I have noticed how powerful list comprehension is. + "items": [ + { + "value": x, + } + for x in lst + ], + } + response = await client.post(url, headers=headers, json=data) + print(f"Uploaded {list_name} to Cloudflare") + if response.status_code != 200: + print(f"Error uploading {list_name}: {response.text}") + exit(1) - data = { - "name": list_name, - "type": "DOMAIN", - "description": "A blocklist of ad domains", - # Writing this program, I have noticed how powerful list comprehension is. - "items": [ - { - "value": x, - } - for x in lst - ], - } - response = requests.post(url, headers=headers, json=data, timeout=10) - print(f"Uploaded {list_name} to Cloudflare") - if response.status_code != 200: - print(f"Error uploading {list_name}: {response.text}") def create_dns_policy(lists, account_id: str, token: str) -> None: @@ -108,7 +113,7 @@ def main(): blocklists = get_blocklists() blocklists = apply_whitelists(blocklists) lists = split_list(blocklists) - upload_to_cloudflare(lists, account_id, token) + asyncio.run(upload_to_cloudflare(lists, account_id, token)) cloud_lists = utils.get_lists(account_id, token) cloud_lists = utils.filter_adblock_lists(cloud_lists) create_dns_policy(cloud_lists, account_id, token)