README.md

@@ -6,7 +6,7 @@ This is a serverless adblocking solution that uses Cloudflare's Zero Trust Gatew
 This project was heavily inspired by [this blog post](https://blog.marcolancini.it/2022/blog-serverless-ad-blocking-with-cloudflare-gateway/)  
 
 
-### Prerequisites
+### Pre-requisites
 * Python > 3.10  
 * A Cloudflare account with Zero Trust enabled  
 * A Cloudflare API tolken with the following permissions:  
@@ -15,11 +15,11 @@ This project was heavily inspired by [this blog post](https://blog.marcolancini.
     * Access: Apps and Policies: Edit  
 * A device with the WARP client installed and configured to use a Zero Trust account  
 
-
+<!-- 
 ### Installation  
 #### From PyPi  
-`pip install cloudflare-gateway-adblocking`  
+`pip install cloudflare-gateway-adblocking`
-
+ -->
 
 ### Usage   
 #### Setting Cloudflare credentials  
@@ -43,6 +43,4 @@ For example:
 #### Deleting blocklists and firewall policy  
 To delete the blocklists from Cloudflare and delete the firewall policy, use the `delete` subcommand.  
 For example:  
-`cloudflare-gateway-adblocking delete`  
+`cloudflare-gateway-adblocking delete`  
-### Help  
-For help, use the `--help` flag.  

poetry.lock

@@ -1,26 +1,5 @@
 # This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand.
 
-[[package]]
-name = "anyio"
-version = "3.7.1"
-description = "High level compatibility layer for multiple asynchronous event loop implementations"
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"},
-    {file = "anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780"},
-]
-
-[package.dependencies]
-exceptiongroup = {version = "*", markers = "python_version < \"3.11\""}
-idna = ">=2.8"
-sniffio = ">=1.1"
-
-[package.extras]
-doc = ["Sphinx", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-jquery"]
-test = ["anyio[trio]", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"]
-trio = ["trio (<0.22)"]
-
 [[package]]
 name = "black"
 version = "23.7.0"
@@ -186,75 +165,6 @@ files = [
     {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
 ]
 
-[[package]]
-name = "exceptiongroup"
-version = "1.1.2"
-description = "Backport of PEP 654 (exception groups)"
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"},
-    {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"},
-]
-
-[package.extras]
-test = ["pytest (>=6)"]
-
-[[package]]
-name = "h11"
-version = "0.14.0"
-description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"},
-    {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"},
-]
-
-[[package]]
-name = "httpcore"
-version = "0.17.3"
-description = "A minimal low-level HTTP client."
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "httpcore-0.17.3-py3-none-any.whl", hash = "sha256:c2789b767ddddfa2a5782e3199b2b7f6894540b17b16ec26b2c4d8e103510b87"},
-    {file = "httpcore-0.17.3.tar.gz", hash = "sha256:a6f30213335e34c1ade7be6ec7c47f19f50c56db36abef1a9dfa3815b1cb3888"},
-]
-
-[package.dependencies]
-anyio = ">=3.0,<5.0"
-certifi = "*"
-h11 = ">=0.13,<0.15"
-sniffio = "==1.*"
-
-[package.extras]
-http2 = ["h2 (>=3,<5)"]
-socks = ["socksio (==1.*)"]
-
-[[package]]
-name = "httpx"
-version = "0.24.1"
-description = "The next generation HTTP client."
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "httpx-0.24.1-py3-none-any.whl", hash = "sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd"},
-    {file = "httpx-0.24.1.tar.gz", hash = "sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd"},
-]
-
-[package.dependencies]
-certifi = "*"
-httpcore = ">=0.15.0,<0.18.0"
-idna = "*"
-sniffio = "*"
-
-[package.extras]
-brotli = ["brotli", "brotlicffi"]
-cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
-http2 = ["h2 (>=3,<5)"]
-socks = ["socksio (==1.*)"]
-
 [[package]]
 name = "idna"
 version = "3.4"
@@ -393,17 +303,6 @@ files = [
     {file = "ruff-0.0.281.tar.gz", hash = "sha256:bab2cdfa78754315cccc2b4d46ad6181aabb29e89747a3b135a4b85e11baa025"},
 ]
 
-[[package]]
-name = "sniffio"
-version = "1.3.0"
-description = "Sniff out which async library your code is running under"
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"},
-    {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"},
-]
-
 [[package]]
 name = "tomli"
 version = "2.0.1"
@@ -449,4 +348,4 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "0c9985ad60dc4fee35917dd0ff673af48019ca19a009b5185f6eff9343428a84"
+content-hash = "d1e33ec5def7b19f6fd09ac80725a9a9f24da3274b075ee37eb53fe2b2f54219"

pyproject.toml

@@ -18,7 +18,6 @@ python = "^3.10"
 requests = "^2.31.0"
 loguru = "^0.7.0"
 python-dotenv = "^1.0.0"
-httpx = "^0.24.1"
 
 
 [tool.poetry.group.dev.dependencies]

src/__main__.py

@@ -5,7 +5,7 @@ import argparse
 import os
 from pathlib import Path
 from sys import exit, stderr
-import asyncio
+
 import dotenv
 from loguru import logger
 
@@ -16,6 +16,20 @@ ACCOUNT_ID = None
 
 
 def main():
+    # Setup logging
+    logger.remove()
+    # ^10 is a formatting directive to center with a padding of 10
+    logger_format = "<green>{time:YYYY-MM-DD HH:mm:ss}</green> |<level>{level: ^10}</level>| <level>{message}</level>"  # noqa E501
+    logger.add(stderr, format=logger_format, colorize=True, level="DEBUG")
+
+    # Load .env if it exists
+    # This must precede the argparse setup as env variables are used as default values
+    if Path(".env").is_file():
+        dotenv.load_dotenv()
+        logger.info("Loaded .env file")
+    else:
+        logger.info("No .env file found")
+
     # Parse arguments
 
     # Set up argparse
@@ -29,20 +43,17 @@ def main():
     credential_args = argparser.add_argument_group("Cloudflare Credentials")
 
     # Add arguments
-
-    # General arguments
-    argparser.add_argument('--log-level', '-l', help='Log level', default='INFO')  # noqa E501
-
-    # Credential arguments
     credential_args.add_argument(
         "--account-id",
         "-a",
         help="Cloudflare account ID - environment variable: CLOUDFLARE_ACCOUNT_ID",
+        default=os.environ.get("CLOUDFLARE_ACCOUNT_ID"),
     )
     credential_args.add_argument(
         "--token",
         "-t",
         help="Cloudflare API token - environment variable: CLOUDFLARE_TOKEN",
+        default=os.environ.get("CLOUDFLARE_TOKEN"),
     )
 
     # Add subcommands
@@ -67,7 +78,7 @@ def main():
         "--whitelists",
         "-w",
         help="Either a whitelist hosts file or a directory containing whitelist hosts files",  # noqa E501
-        default="whitelists",  # Not really needed because the apply_whitelists function will default to this  # noqa: E501
+        default="whitelist.txt",  # Not really needed because the apply_whitelists function will default to this  # noqa: E501
     )
     # Add subcommand: delete
     delete_parser = subparsers.add_parser(
@@ -77,39 +88,22 @@ def main():
 
     args = argparser.parse_args()
 
-    # Set up logging
-    set_primary_logger(args.log_level)
     logger.debug(args)
 
-
-
     # Load variables
     global TOKEN
     global ACCOUNT_ID
     TOKEN = args.token
     ACCOUNT_ID = args.account_id
-
+    # Check if variables are set
-    # Check if credentials are set, if they are not, attempt to load from environment variables and .env  # noqa E501
     if TOKEN is None or ACCOUNT_ID is None:
-        logger.info("No credentials provided with flags")
+        logger.error(
-        if Path(".env").is_file():
+            "No environment variables found. Please create a .env file or .envrc file"
-            logger.debug("Loading .env")
+        )  # noqa E501
-            dotenv.load_dotenv()
+        exit(1)
-        else:
-            logger.debug("No .env file found")
-        try:
-            TOKEN = os.environ["CLOUDFLARE_TOKEN"]
-            ACCOUNT_ID = os.environ["CLOUDFLARE_ACCOUNT_ID"]
-            logger.info("Loaded credentials from environment variables")
-        except KeyError:
-            logger.error("No credentials provided")
-            argparser.print_help()
-            exit(1)
-
     try:
         args.func(args)
     except AttributeError as e:
-        logger.debug(e)
         logger.error("No subcommand specified")
         argparser.print_help()
         exit(1)
@@ -120,7 +114,7 @@ def upload_to_cloudflare(args):
     blocklists = upload.get_blocklists(args.blocklists)
     blocklists = upload.apply_whitelists(blocklists, args.whitelists)
     lists = upload.split_list(blocklists)
-    asyncio.run(upload.upload_to_cloudflare(lists, ACCOUNT_ID, TOKEN))
+    upload.upload_to_cloudflare(lists, ACCOUNT_ID, TOKEN)
     cloud_lists = utils.get_lists(ACCOUNT_ID, TOKEN)
     cloud_lists = utils.filter_adblock_lists(cloud_lists)
     upload.create_dns_policy(cloud_lists, ACCOUNT_ID, TOKEN)
@@ -132,13 +126,7 @@ def delete_from_cloudflare(args):
     delete.delete_adblock_policy(rules, ACCOUNT_ID, TOKEN)
     lists = utils.get_lists(ACCOUNT_ID, TOKEN)
     lists = utils.filter_adblock_lists(lists)
-    asyncio.run(delete.delete_adblock_list(lists, ACCOUNT_ID, TOKEN))
+    delete.delete_adblock_list(lists, ACCOUNT_ID, TOKEN)
-
-def set_primary_logger(log_level):
-    logger.remove()
-    # ^10 is a formatting directive to center with a padding of 10
-    logger_format = "<green>{time:YYYY-MM-DD HH:mm:ss}</green> |<level>{level: ^10}</level>| <level>{message}</level>"  # noqa E501
-    logger.add(stderr, format=logger_format, colorize=True, level=log_level)
 
 
 if __name__ == "__main__":

src/utils/delete.py

@@ -1,26 +1,23 @@
 # This is a scriprt to undo the changes made by adblock-zerotrust.py
 
 import requests
-import httpx 
+
-import asyncio
 from . import utils
 
 
-async def delete_adblock_list(lists: dict, account_id: str, token: str):
+def delete_adblock_list(lists: dict, account_id: str, token: str):
     try:
-        async with httpx.AsyncClient() as client:
+        for lst in lists:
-            for lst in lists:
+            url = f'https://api.cloudflare.com/client/v4/accounts/{account_id}/gateway/lists/{lst["id"]}'
-                url = f'https://api.cloudflare.com/client/v4/accounts/{account_id}/gateway/lists/{lst["id"]}'
+            headers = {
-                headers = {
+                "Authorization": f"Bearer {token}",
-                    "Authorization": f"Bearer {token}",
+                "Content-Type": "application/json",
-                    "Content-Type": "application/json",
+            }
-                }
+            response = requests.delete(url, headers=headers, timeout=10)
-                # response = requests.delete(url, headers=headers, timeout=10)
+            if response.status_code != 200:
-                response = await client.delete(url, headers=headers, timeout=10)
+                print(f"Error deleting list: {response.text}")
-                if response.status_code != 200:
+            else:
-                    print(f"Error deleting list: {response.text}")
+                print(f'Deleted list {lst["name"]}')
-                else:
-                    print(f'Deleted list {lst["name"]}')
     except TypeError as e:
         if str(e) == "'NoneType' object is not iterable":
             print("No lists found")
@@ -51,7 +48,7 @@ def main():
     token = input("Enter your Cloudflare API token: ")
 
     rules = utils.get_gateway_rules(account_id, token)
-    asyncio.run(utils.delete_adblock_list(rules, account_id, token))
+    delete_adblock_policy(rules, account_id, token)
     lists = utils.get_lists(account_id, token)
     lists = utils.filter_adblock_lists(lists)
     delete_adblock_list(lists, account_id, token)

src/utils/upload.py

@@ -1,8 +1,6 @@
 import pathlib
 
 import requests
-import asyncio
-import httpx
 
 from . import utils
 
@@ -48,36 +46,33 @@ def split_list(blocklists):
     return lists
 
 
-async def upload_to_cloudflare(lists, account_id: str, token: str) -> None: 
+def upload_to_cloudflare(lists, account_id: str, token: str) -> None:
-    async with httpx.AsyncClient() as client:
+    for i, lst in enumerate(lists):
-        for i, lst in enumerate(lists):
+        list_name = f"adblock-list-{i + 1}"
-            list_name = f"adblock-list-{i + 1}"
+        url = (
-            url = (
+            f"https://api.cloudflare.com/client/v4/accounts/{account_id}/gateway/lists"
-                f"https://api.cloudflare.com/client/v4/accounts/{account_id}/gateway/lists"
+        )
-            )
+        headers = {
-            headers = {
+            "Authorization": f"Bearer {token}",
-                "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
-                "Content-Type": "application/json",
+        }
-            }
-
-            data = {
-                "name": list_name,
-                "type": "DOMAIN",
-                "description": "A blocklist of ad domains",
-                # Writing this program, I have noticed how powerful list comprehension is.
-                "items": [
-                    {
-                        "value": x,
-                    }
-                    for x in lst
-                ],
-            }
-            response = await client.post(url, headers=headers, json=data)
-            print(f"Uploaded {list_name} to Cloudflare")
-            if response.status_code != 200:
-                print(f"Error uploading {list_name}: {response.text}")
-                exit(1)
 
+        data = {
+            "name": list_name,
+            "type": "DOMAIN",
+            "description": "A blocklist of ad domains",
+            # Writing this program, I have noticed how powerful list comprehension is.
+            "items": [
+                {
+                    "value": x,
+                }
+                for x in lst
+            ],
+        }
+        response = requests.post(url, headers=headers, json=data, timeout=10)
+        print(f"Uploaded {list_name} to Cloudflare")
+        if response.status_code != 200:
+            print(f"Error uploading {list_name}: {response.text}")
 
 
 def create_dns_policy(lists, account_id: str, token: str) -> None:
@@ -113,7 +108,7 @@ def main():
     blocklists = get_blocklists()
     blocklists = apply_whitelists(blocklists)
     lists = split_list(blocklists)
-    asyncio.run(upload_to_cloudflare(lists, account_id, token))
+    upload_to_cloudflare(lists, account_id, token)
     cloud_lists = utils.get_lists(account_id, token)
     cloud_lists = utils.filter_adblock_lists(cloud_lists)
     create_dns_policy(cloud_lists, account_id, token)

src/utils/utils.py

@@ -67,7 +67,7 @@ def convert_to_list(file: pathlib.Path) -> list:
             "ip6-allnodes",
             "ip6-allrouters",
             "ip6-allhosts",
-            "0.0.0.0",  # skipcq: BAN-B104
+            "0.0.0.0",
         ]
         matches = [
             re.search(r"^(?:127\.0\.0\.1|0\.0\.0\.0|::1)\s+(.+?)(?:\s+#.+)?$", line)
@@ -78,7 +78,7 @@ def convert_to_list(file: pathlib.Path) -> list:
             for match in matches
             if match and match.group(1) not in loopback
         ]
-        # print(f"First 5 hosts: {hosts[:5]}")
+        print(f"First 5 hosts: {hosts[:5]}")
         return hosts