Delete utils.py

Also added information on whitelists to `README.md`

.deepsource.toml

@@ -0,0 +1,13 @@
+version = 1
+
+[[analyzers]]
+name = "python"
+
+  [analyzers.meta]
+  runtime_version = "3.x.x"
+
+[[transformers]]
+name = "black"
+
+[[transformers]]
+name = "isort"

.env.example

@@ -1,2 +1,2 @@
-export CLOUDFLARE_ACCOUNT_ID=
-export CLOUDFLARE_TOKEN=
+CLOUDFLARE_ACCOUNT_ID=
+CLOUDFLARE_TOKEN=

.gitignore

@@ -8,3 +8,4 @@ tmp.py
 .venv
 dist/
 .ruff_cache/
+hosts.txt

.vscode/settings.json

@@ -1,4 +1,10 @@
 {
-    "python.languageServer": "Pylance",
-    "python.analysis.ignore": [ "*" ] // Ruff is used for linting but Pylance still is useful
+    "python.languageServer": "Pylance", // Ruff is used for linting but Pylance still is useful for intellisense
+    "python.analysis.ignore": [
+        "*"
+    ],
+    "python.analysis.exclude": [
+        "."
+    ],
+    "python.linting.enabled": false // https://github.com/microsoft/vscode-python/wiki/Migration-to-Python-Tools-Extensions
 }

README.md

@@ -1,2 +1,46 @@
 # Cloudflare Gateway Adblocking  
 Serverless adblocking via Cloudflare Zero Trust Gateway  
+
+### What is this?  
+This is a serverless adblocking solution that uses Cloudflare's Zero Trust Gateway to block ads by parsing a hosts file and creating a firewall rule to block the domains. It can be used as an alternative to Pi-Hole or other adblocking solutions.  
+This project was heavily inspired by [this blog post](https://blog.marcolancini.it/2022/blog-serverless-ad-blocking-with-cloudflare-gateway/)  
+
+
+### Pre-requisites
+* Python > 3.10  
+* A Cloudflare account with Zero Trust enabled  
+* A Cloudflare API tolken with the following permissions:  
+    * Zero Trust: Edit  
+    * Account Firewall Access Rules: Edit  
+    * Access: Apps and Policies: Edit  
+* A device with the WARP client installed and configured to use a Zero Trust account  
+
+<!-- 
+### Installation  
+#### From PyPi  
+`pip install cloudflare-gateway-adblocking`
+ -->
+
+### Usage   
+#### Setting Cloudflare credentials  
+##### Environment variables  
+The following environment variables can be used to set the Cloudflare credentials:  
+* `CLOUDFLARE_ACCOUNT_ID`
+* `CLOUDFLARE_TOKEN`  
+These can either be set in the environment or in a `.env` file in the current working directory.  
+#### Command line flags  
+The following command line flags can be used to set the Cloudflare credentials:
+* Cloudflare Account ID: `--account-id` / `-a`  
+* Cloudflare Token: `--token` / `-t`  
+#### Passing blocklists  
+Blocklists can be passed to the program via the command line flag `--blocklist` / `-b`. This flag can either point to a hosts file or a directory containing hosts files. If this flag is not passed, the program will look for a file or directory named `blocklists` in the current working directory.  
+# Passing whitelists  
+Whitelists can be passed to the program via the command line flag `--whitelist` / `-w`. This flag can either point to a hosts file or a directory containing hosts files. If this flag is not passed, then if a file or directory named `whitelists` exists in the current working directory, it will be used. Domains in this whitelist will be excluded from the blocklists.  
+#### Uploading blocklists and creating a firewall policy
+To upload the blocklists to Cloudflare and create a firewall policy, use the `upload` subcommand.  
+For example:  
+`cloudflare-gateway-adblocking upload`  
+#### Deleting blocklists and firewall policy  
+To delete the blocklists from Cloudflare and delete the firewall policy, use the `delete` subcommand.  
+For example:  
+`cloudflare-gateway-adblocking delete`  

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cloudflare-gateway-adblocking"
-version = "0.1.9"
+version = "0.1.0"
 description = "Serverless adblocking via Cloudflare Zero Trust Gateway"
 authors = ["slastechno <77907286+slashtechno@users.noreply.github.com>"]
 license = "MIT"

src/__main__.py

@@ -1,13 +1,15 @@
 # To run from the root project directory, run the following command:
 # python -m src.__main__
-from loguru import logger
-from .utils import upload, delete, utils
-
+# python -m src # also works because __main__ is the default module
 import argparse
 import os
-import dotenv
-from sys import exit, stderr
 from pathlib import Path
+from sys import exit, stderr
+
+import dotenv
+from loguru import logger
+
+from .utils import delete, upload, utils
 
 TOKEN = None
 ACCOUNT_ID = None
@@ -70,13 +72,13 @@ def main():
         "--blocklists",
         "-b",
         help="Either a blocklist hosts file or a directory containing blocklist hosts files",  # noqa E501
-        default="blocklists",
+        default="blocklists",  # Not really needed because the get_blocklists function will default to this  # noqa: E501
     )
     upload_parser.add_argument(
         "--whitelists",
         "-w",
         help="Either a whitelist hosts file or a directory containing whitelist hosts files",  # noqa E501
-        default="whitelist.txt",  # Need to change this so it's optional
+        default="whitelist.txt",  # Not really needed because the apply_whitelists function will default to this  # noqa: E501
     )
     # Add subcommand: delete
     delete_parser = subparsers.add_parser(
@@ -102,8 +104,9 @@ def main():
     try:
         args.func(args)
     except AttributeError as e:
-        logger.debug(e)
+        logger.error("No subcommand specified")
         argparser.print_help()
+        exit(1)
 
 
 def upload_to_cloudflare(args):

src/utils/delete.py

@@ -1,6 +1,7 @@
 # This is a scriprt to undo the changes made by adblock-zerotrust.py
 
 import requests
+
 from . import utils
 
 
@@ -43,11 +44,14 @@ def delete_adblock_policy(policies: dict, account_id: str, token: str):
 
 
 def main():
-    rules = utils.get_gateway_rules()
-    delete_adblock_policy(rules)
-    lists = utils.get_lists()
+    account_id = input("Enter your Cloudflare account ID: ")
+    token = input("Enter your Cloudflare API token: ")
+
+    rules = utils.get_gateway_rules(account_id, token)
+    delete_adblock_policy(rules, account_id, token)
+    lists = utils.get_lists(account_id, token)
     lists = utils.filter_adblock_lists(lists)
-    delete_adblock_list(lists)
+    delete_adblock_list(lists, account_id, token)
 
 
 if __name__ == "__main__":

src/utils/upload.py

@@ -1,9 +1,11 @@
-import requests
-from . import utils
 import pathlib
 
+import requests
 
-def get_blocklists(hosts_path: str = None):
+from . import utils
+
+
+def get_blocklists(hosts_path: str = "blocklists"):
     blocklists = []
     hosts_path = pathlib.Path(hosts_path)
     if hosts_path.is_file():
@@ -16,7 +18,7 @@ def get_blocklists(hosts_path: str = None):
     return blocklists
 
 
-def apply_whitelists(blocklists, whitelist: str = None):
+def apply_whitelists(blocklists, whitelist: str = "whitelists"):
     # If whitelist is a file, convert it to a list.
     # If whitelist is a directory, convert all files in it to a list and combine them.
     # If it does not exist, return the original blocklists
@@ -100,13 +102,16 @@ def create_dns_policy(lists, account_id: str, token: str) -> None:
 
 
 def main():
+    account_id = input("Enter your Cloudflare account ID: ")
+    token = input("Enter your Cloudflare API token: ")
+
     blocklists = get_blocklists()
     blocklists = apply_whitelists(blocklists)
     lists = split_list(blocklists)
-    upload_to_cloudflare(lists)
-    cloud_lists = utils.get_lists()
+    upload_to_cloudflare(lists, account_id, token)
+    cloud_lists = utils.get_lists(account_id, token)
     cloud_lists = utils.filter_adblock_lists(cloud_lists)
-    create_dns_policy(cloud_lists)
+    create_dns_policy(cloud_lists, account_id, token)
 
 
 if __name__ == "__main__":

src/utils/utils.py

@@ -1,4 +1,5 @@
 import pathlib
+import re
 
 import requests
 
@@ -53,23 +54,34 @@ class Config:
 # Convert a hosts file to a simple hostname list
 def convert_to_list(file: pathlib.Path) -> list:
     with open(file, "r") as f:
-        # Don't read commented lines; strip whitespace;
-        #  remove 127.0.0.1 from beginning of line;
-        # ignore lines with "localhost"; ignore lines with "::1";
-        # ignore newlines
-        hosts = [
-            i[10:].strip()
-            for i in f.readlines()
-            if not i.startswith("#") and "localhost" not in i and "::1" not in i
+        # Loop through the file and using regex, only get the domain names
+        # Remove the prefixed loopback domain and suffixed comments
+        # Remove any empty strings
+        loopback = [
+            "localhost",
+            "::1",
+            "localhost.localdomain",
+            "broadcasthost",
+            "local",
+            "ip6-localhost",
+            "ip6-loopback",
+            "ip6-localnet",
+            "ip6-mcastprefix",
+            "ip6-allnodes",
+            "ip6-allrouters",
+            "ip6-allhosts",
+            "0.0.0.0",
         ]
-        # Equivalent to:
-        # for x in f.readlines():
-        #     if not x.startswith('#') and 'localhost' not in x and '::1' not in x:
-        #         hosts.append(x[10:].strip())
-
-        # If there are any empty strings in the list, remove them
-        # For some reason, whitelist seems to still be present
-        hosts = [i for i in hosts if i != ""]
+        matches = [
+            re.search(r"^(?:127\.0\.0\.1|0\.0\.0\.0|::1)\s+(.+?)(?:\s+#.+)?$", line)
+            for line in f
+        ]
+        hosts = [
+            match.group(1)
+            for match in matches
+            if match and match.group(1) not in loopback
+        ]
+        print(f"First 5 hosts: {hosts[:5]}")
         return hosts