slashtechno
/
cloudflare-gateway-adblocking
mirror of https://github.com/slashtechno/cloudflare-gateway-adblocking.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add basic `async` support (#6)

This commit is contained in:
/techno 2023-08-08 15:50:02 -04:00 committed by GitHub
parent c64e2e139d
commit 50683e1543
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 154 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
poetry.lock generated
View File

@ -1,5 +1,26 @@
# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. # This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand.
[[package]]
name = "anyio"
version = "3.7.1"
description = "High level compatibility layer for multiple asynchronous event loop implementations"
optional = false
python-versions = ">=3.7"
files = [
{file = "anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"},
{file = "anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780"},
]
[package.dependencies]
exceptiongroup = {version = "*", markers = "python_version < \"3.11\""}
idna = ">=2.8"
sniffio = ">=1.1"
[package.extras]
doc = ["Sphinx", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-jquery"]
test = ["anyio[trio]", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"]
trio = ["trio (<0.22)"]
[[package]] [[package]]
name = "black" name = "black"
version = "23.7.0" version = "23.7.0"
@ -165,6 +186,75 @@ files = [
{file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
] ]
[[package]]
name = "exceptiongroup"
version = "1.1.2"
description = "Backport of PEP 654 (exception groups)"
optional = false
python-versions = ">=3.7"
files = [
{file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"},
{file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"},
]
[package.extras]
test = ["pytest (>=6)"]
[[package]]
name = "h11"
version = "0.14.0"
description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
optional = false
python-versions = ">=3.7"
files = [
{file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"},
{file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"},
]
[[package]]
name = "httpcore"
version = "0.17.3"
description = "A minimal low-level HTTP client."
optional = false
python-versions = ">=3.7"
files = [
{file = "httpcore-0.17.3-py3-none-any.whl", hash = "sha256:c2789b767ddddfa2a5782e3199b2b7f6894540b17b16ec26b2c4d8e103510b87"},
{file = "httpcore-0.17.3.tar.gz", hash = "sha256:a6f30213335e34c1ade7be6ec7c47f19f50c56db36abef1a9dfa3815b1cb3888"},
]
[package.dependencies]
anyio = ">=3.0,<5.0"
certifi = "*"
h11 = ">=0.13,<0.15"
sniffio = "==1.*"
[package.extras]
http2 = ["h2 (>=3,<5)"]
socks = ["socksio (==1.*)"]
[[package]]
name = "httpx"
version = "0.24.1"
description = "The next generation HTTP client."
optional = false
python-versions = ">=3.7"
files = [
{file = "httpx-0.24.1-py3-none-any.whl", hash = "sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd"},
{file = "httpx-0.24.1.tar.gz", hash = "sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd"},
]
[package.dependencies]
certifi = "*"
httpcore = ">=0.15.0,<0.18.0"
idna = "*"
sniffio = "*"
[package.extras]
brotli = ["brotli", "brotlicffi"]
cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
http2 = ["h2 (>=3,<5)"]
socks = ["socksio (==1.*)"]
[[package]] [[package]]
name = "idna" name = "idna"
version = "3.4" version = "3.4"
@ -303,6 +393,17 @@ files = [
{file = "ruff-0.0.281.tar.gz", hash = "sha256:bab2cdfa78754315cccc2b4d46ad6181aabb29e89747a3b135a4b85e11baa025"}, {file = "ruff-0.0.281.tar.gz", hash = "sha256:bab2cdfa78754315cccc2b4d46ad6181aabb29e89747a3b135a4b85e11baa025"},
] ]
[[package]]
name = "sniffio"
version = "1.3.0"
description = "Sniff out which async library your code is running under"
optional = false
python-versions = ">=3.7"
files = [
{file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"},
{file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"},
]
[[package]] [[package]]
name = "tomli" name = "tomli"
version = "2.0.1" version = "2.0.1"
@ -348,4 +449,4 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"]
[metadata] [metadata]
lock-version = "2.0" lock-version = "2.0"
python-versions = "^3.10" python-versions = "^3.10"
content-hash = "d1e33ec5def7b19f6fd09ac80725a9a9f24da3274b075ee37eb53fe2b2f54219" content-hash = "0c9985ad60dc4fee35917dd0ff673af48019ca19a009b5185f6eff9343428a84"

1
pyproject.toml
View File

@ -18,6 +18,7 @@ python = "^3.10"
requests = "^2.31.0" requests = "^2.31.0"
loguru = "^0.7.0" loguru = "^0.7.0"
python-dotenv = "^1.0.0" python-dotenv = "^1.0.0"
httpx = "^0.24.1"
[tool.poetry.group.dev.dependencies] [tool.poetry.group.dev.dependencies]

7
src/__main__.py
View File

@ -2,6 +2,7 @@
# python -m src.__main__ # python -m src.__main__
# python -m src # also works because __main__ is the default module # python -m src # also works because __main__ is the default module
import argparse import argparse
import asyncio
import os import os
from pathlib import Path from pathlib import Path
from sys import exit, stderr from sys import exit, stderr
@ -69,7 +70,7 @@ def main():
"--whitelists", "--whitelists",
"-w", "-w",
help="Either a whitelist hosts file or a directory containing whitelist hosts files", # noqa E501 help="Either a whitelist hosts file or a directory containing whitelist hosts files", # noqa E501
default="whitelist.txt", # Not really needed because the apply_whitelists function will default to this # noqa: E501 default="whitelists", # Not really needed because the apply_whitelists function will default to this # noqa: E501
) )
# Add subcommand: delete # Add subcommand: delete
delete_parser = subparsers.add_parser( delete_parser = subparsers.add_parser(
@ -119,7 +120,7 @@ def upload_to_cloudflare(args):
blocklists = upload.get_blocklists(args.blocklists) blocklists = upload.get_blocklists(args.blocklists)
blocklists = upload.apply_whitelists(blocklists, args.whitelists) blocklists = upload.apply_whitelists(blocklists, args.whitelists)
lists = upload.split_list(blocklists) lists = upload.split_list(blocklists)
upload.upload_to_cloudflare(lists, ACCOUNT_ID, TOKEN) asyncio.run(upload.upload_to_cloudflare(lists, ACCOUNT_ID, TOKEN))
cloud_lists = utils.get_lists(ACCOUNT_ID, TOKEN) cloud_lists = utils.get_lists(ACCOUNT_ID, TOKEN)
cloud_lists = utils.filter_adblock_lists(cloud_lists) cloud_lists = utils.filter_adblock_lists(cloud_lists)
upload.create_dns_policy(cloud_lists, ACCOUNT_ID, TOKEN) upload.create_dns_policy(cloud_lists, ACCOUNT_ID, TOKEN)
@ -131,7 +132,7 @@ def delete_from_cloudflare(args):
delete.delete_adblock_policy(rules, ACCOUNT_ID, TOKEN) delete.delete_adblock_policy(rules, ACCOUNT_ID, TOKEN)
lists = utils.get_lists(ACCOUNT_ID, TOKEN) lists = utils.get_lists(ACCOUNT_ID, TOKEN)
lists = utils.filter_adblock_lists(lists) lists = utils.filter_adblock_lists(lists)
delete.delete_adblock_list(lists, ACCOUNT_ID, TOKEN) asyncio.run(delete.delete_adblock_list(lists, ACCOUNT_ID, TOKEN))
def set_primary_logger(log_level): def set_primary_logger(log_level):

31
src/utils/delete.py
View File

@ -1,23 +1,28 @@
# This is a scriprt to undo the changes made by adblock-zerotrust.py # This is a scriprt to undo the changes made by adblock-zerotrust.py
import asyncio
import httpx
import requests import requests
from . import utils from . import utils
def delete_adblock_list(lists: dict, account_id: str, token: str): async def delete_adblock_list(lists: dict, account_id: str, token: str):
try: try:
for lst in lists: async with httpx.AsyncClient() as client:
url = f'https://api.cloudflare.com/client/v4/accounts/{account_id}/gateway/lists/{lst["id"]}' for lst in lists:
headers = { url = f'https://api.cloudflare.com/client/v4/accounts/{account_id}/gateway/lists/{lst["id"]}'
"Authorization": f"Bearer {token}", headers = {
"Content-Type": "application/json", "Authorization": f"Bearer {token}",
} "Content-Type": "application/json",
response = requests.delete(url, headers=headers, timeout=10) }
if response.status_code != 200: # response = requests.delete(url, headers=headers, timeout=10)
print(f"Error deleting list: {response.text}") response = await client.delete(url, headers=headers, timeout=10)
else: if response.status_code != 200:
print(f'Deleted list {lst["name"]}') print(f"Error deleting list: {response.text}")
else:
print(f'Deleted list {lst["name"]}')
except TypeError as e: except TypeError as e:
if str(e) == "'NoneType' object is not iterable": if str(e) == "'NoneType' object is not iterable":
print("No lists found") print("No lists found")
@ -48,7 +53,7 @@ def main():
token = input("Enter your Cloudflare API token: ") token = input("Enter your Cloudflare API token: ")
rules = utils.get_gateway_rules(account_id, token) rules = utils.get_gateway_rules(account_id, token)
delete_adblock_policy(rules, account_id, token) asyncio.run(utils.delete_adblock_list(rules, account_id, token))
lists = utils.get_lists(account_id, token) lists = utils.get_lists(account_id, token)
lists = utils.filter_adblock_lists(lists) lists = utils.filter_adblock_lists(lists)
delete_adblock_list(lists, account_id, token) delete_adblock_list(lists, account_id, token)

56
src/utils/upload.py
View File

@ -1,5 +1,7 @@
import asyncio
import pathlib import pathlib
import httpx
import requests import requests
from . import utils from . import utils
@ -46,33 +48,33 @@ def split_list(blocklists):
return lists return lists
def upload_to_cloudflare(lists, account_id: str, token: str) -> None: async def upload_to_cloudflare(lists, account_id: str, token: str) -> None:
for i, lst in enumerate(lists): async with httpx.AsyncClient() as client:
list_name = f"adblock-list-{i + 1}" for i, lst in enumerate(lists):
url = ( list_name = f"adblock-list-{i + 1}"
f"https://api.cloudflare.com/client/v4/accounts/{account_id}/gateway/lists" url = f"https://api.cloudflare.com/client/v4/accounts/{account_id}/gateway/lists"
) headers = {
headers = { "Authorization": f"Bearer {token}",
"Authorization": f"Bearer {token}", "Content-Type": "application/json",
"Content-Type": "application/json", }
}
data = { data = {
"name": list_name, "name": list_name,
"type": "DOMAIN", "type": "DOMAIN",
"description": "A blocklist of ad domains", "description": "A blocklist of ad domains",
# Writing this program, I have noticed how powerful list comprehension is. # Writing this program, I have noticed how powerful list comprehension is.
"items": [ "items": [
{ {
"value": x, "value": x,
} }
for x in lst for x in lst
], ],
} }
response = requests.post(url, headers=headers, json=data, timeout=10) response = await client.post(url, headers=headers, json=data)
print(f"Uploaded {list_name} to Cloudflare") print(f"Uploaded {list_name} to Cloudflare")
if response.status_code != 200: if response.status_code != 200:
print(f"Error uploading {list_name}: {response.text}") print(f"Error uploading {list_name}: {response.text}")
exit(1)
def create_dns_policy(lists, account_id: str, token: str) -> None: def create_dns_policy(lists, account_id: str, token: str) -> None:
@ -108,7 +110,7 @@ def main():
blocklists = get_blocklists() blocklists = get_blocklists()
blocklists = apply_whitelists(blocklists) blocklists = apply_whitelists(blocklists)
lists = split_list(blocklists) lists = split_list(blocklists)
upload_to_cloudflare(lists, account_id, token) asyncio.run(upload_to_cloudflare(lists, account_id, token))
cloud_lists = utils.get_lists(account_id, token) cloud_lists = utils.get_lists(account_id, token)
cloud_lists = utils.filter_adblock_lists(cloud_lists) cloud_lists = utils.filter_adblock_lists(cloud_lists)
create_dns_policy(cloud_lists, account_id, token) create_dns_policy(cloud_lists, account_id, token)