emdee/exclude_badExits
1
0
Fork 0
Code Issues Pull requests Releases Packages Wiki Activity

pep8 isort

Browse source
This commit is contained in:
emdee 2022-11-17 08:58:45 +00:00
parent d6200d6302
commit aac3793b35
3 changed files with 192 additions and 188 deletions
Download patch file Download diff file Expand all files Collapse all files

218
exclude_badExits.py
View file

@ -79,24 +79,21 @@ For usage, do ```python3 exclude_badExits.py --help`
"""
import sys
import os
import re
import socket
import time
import argparse
import string
import os
import sys
import time
from io import StringIO
import ipaddress
# list(ipaddress._find_address_range(ipaddress.IPv4Network('172.16.0.0/12'))
from urllib3.util.ssl_match_hostname import CertificateError
import stem
import urllib3
from stem import InvalidRequest
from stem.control import Controller
from stem.connection import IncorrectPassword
from stem.util.tor_tools import is_valid_fingerprint
from urllib3.util.ssl_match_hostname import CertificateError
# list(ipaddress._find_address_range(ipaddress.IPv4Network('172.16.0.0/12'))
try:
from ruamel.yaml import YAML
yaml = YAML(typ='rt')
@ -112,20 +109,24 @@ if yaml is None:
yaml = None
try:
from unbound import ub_ctx,RR_TYPE_TXT,RR_CLASS_IN
from unbound import RR_CLASS_IN, RR_TYPE_TXT, ub_ctx
except:
ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None
global LOG
import logging
import warnings
warnings.filterwarnings('ignore')
LOG = logging.getLogger()
from support_onions import (bAreWeConnected, icheck_torrc, lIntroductionPoints,
oGetStemController, vwait_for_controller,
yKNOWN_NODNS, zResolveDomain)
from support_phantompy import vsetup_logging
from trustor_poc import TrustorError, idns_validate
from trustor_poc import oDownloadUrlUrllib3 as oDownloadUrl
from trustor_poc import idns_validate, TrustorError
from support_onions import icheck_torrc, bAreWeConnected, lIntroductionPoints, zResolveDomain, vwait_for_controller, yKNOWN_NODNS
LOG.info("imported HTTPSAdapter")
ETC_DIR = '/etc/tor/yaml'
@ -138,17 +139,6 @@ sEXCLUDE_EXIT_KEY = 'ExcludeNodes'
sINCLUDE_EXIT_KEY = 'ExitNodes'
sINCLUDE_GUARD_KEY = 'EntryNodes'
def oMakeController(sSock='', port=9051):
import getpass
if sSock and os.path.exists(sSock):
controller = Controller.from_socket_file(path=sSock)
else:
controller = Controller.from_port(port=port)
sys.stdout.flush()
p = getpass.unix_getpass(prompt='Controller Password: ', stream=sys.stderr)
controller.authenticate(p)
return controller
oBAD_NODES = {}
oBAD_ROOT = 'BadNodes'
oBAD_NODES[oBAD_ROOT] = {}
@ -163,8 +153,8 @@ def lYamlBadNodes(sFile,
global lKNOWN_NODNS
global lMAYBE_NODNS
l = []
if not yaml: return l
if not yaml:
return []
if os.path.exists(sFile):
with open(sFile, 'rt') as oFd:
oBAD_NODES = safe_load(oFd)
@ -188,7 +178,6 @@ oGOOD_NODES = {}
oGOOD_ROOT = 'GoodNodes'
def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'):
global oGOOD_NODES
root = oGOOD_ROOT
l = []
if not yaml: return l
if os.path.exists(sFile):
@ -220,18 +209,18 @@ def bdomain_is_bad(domain, fp):
tBAD_URLS = set()
lATS = ['abuse', 'email']
lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory']
lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone',
'sandbox', 'offlinemasterkey']
lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone',
'sandbox', 'offlinemasterkey']
def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050):
global tBAD_URLS
global lKNOWN_NODNS
# cleanups for yaml
# cleanups
for elt in lINTS:
if elt in a:
a[elt] = int(a[elt])
for elt in lBOOLS:
if elt in a:
if a[elt] in ['y','yes', 'true', 'True']:
if a[elt] in ['y', 'yes', 'true', 'True']:
a[elt] = True
else:
a[elt] = False
@ -291,7 +280,7 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
if a['proof'] not in ['uri-rsa']:
# only support uri for now
if False and ub_ctx:
fp_domain = fp +'.'+domain
fp_domain = fp + '.' + domain
if idns_validate(fp_domain,
libunbound_resolv_file='resolv.conf',
dnssec_DS_file='dnssec-root-trust',
@ -301,7 +290,7 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
return a
LOG.debug(f"{len(keys)} contact fields for {fp}")
url="https://"+domain+"/.well-known/tor-relay/rsa-fingerprint.txt"
url = f"https://{domain}/.well-known/tor-relay/rsa-fingerprint.txt"
try:
LOG.debug(f"Downloading from {domain} for {fp}")
o = oDownloadUrl(url, https_cafile,
@ -319,7 +308,10 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
else:
LOG.warn(f"TrustorError downloading from {domain} {e.args}")
tBAD_URLS.add(a['url'])
except (BaseException ) as e:
except urllib3.exceptions.MaxRetryError as e: # noqa
# maybe offline - not bad
LOG.warn(f"MaxRetryError downloading from {domain} {e}")
except (BaseException) as e:
LOG.error(f"Exception {type(e)} downloading from {domain} {e}")
else:
if hasattr(o, 'status'):
@ -344,7 +336,7 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
if not l:
LOG.warn(f"Downloading from {domain} empty for {fp}")
else:
a['fps'] = [elt for elt in l if elt and len(elt) == 40
a['fps'] = [elt for elt in l if elt and len(elt) == 40 \
and not elt.startswith('#')]
LOG.info(f"Downloaded from {domain} {len(a['fps'])} FPs")
return a
@ -379,7 +371,7 @@ def aParseContact(contact, fp):
if ':' in line]
LOG.debug(f"{fp} {len(l)} fields")
s = f'"{fp}":\n'
s += '\n'.join([f" {line}\"".replace(':',': \"', 1)
s += '\n'.join([f" {line}\"".replace(':', ': \"', 1)
for line in l])
oFd = StringIO(s)
a = safe_load(oFd)
@ -434,8 +426,7 @@ def oMainArgparser(_=None):
default=os.path.join(ETC_DIR, 'badcontacts.yaml'),
help="Yaml file of bad contacts that bad FPs are using")
parser.add_argument('--strict_nodes', type=int, default=0,
choices=[0,1],
parser.add_argument('--strict_nodes', type=int, default=0, choices=[0, 1],
help="Set StrictNodes: 1 is less anonymous but more secure, although some sites may be unreachable")
parser.add_argument('--wait_boot', type=int, default=120,
help="Seconds to wait for Tor to booststrap")
@ -456,29 +447,29 @@ def oMainArgparser(_=None):
help="Write the proof data of the included nodes to a YAML file")
return parser
def vwrite_badnodes(oArgs, oBAD_NODES, slen):
if oArgs.bad_nodes:
tmp = oArgs.bad_nodes +'.tmp'
bak = oArgs.bad_nodes +'.bak'
def vwrite_badnodes(oargs, oBAD_NODES, slen):
if oargs.bad_nodes:
tmp = oargs.bad_nodes +'.tmp'
bak = oargs.bad_nodes +'.bak'
with open(tmp, 'wt') as oFYaml:
yaml.dump(oBAD_NODES, oFYaml)
LOG.info(f"Wrote {slen} to {oArgs.bad_nodes}")
LOG.info(f"Wrote {slen} to {oargs.bad_nodes}")
oFYaml.close()
if os.path.exists(oArgs.bad_nodes):
os.rename(oArgs.bad_nodes, bak)
os.rename(tmp, oArgs.bad_nodes)
if os.path.exists(oargs.bad_nodes):
os.rename(oargs.bad_nodes, bak)
os.rename(tmp, oargs.bad_nodes)
def vwrite_goodnodes(oArgs, oGOOD_NODES, ilen):
if oArgs.good_nodes:
tmp = oArgs.good_nodes +'.tmp'
bak = oArgs.good_nodes +'.bak'
def vwrite_goodnodes(oargs, oGOOD_NODES, ilen):
if oargs.good_nodes:
tmp = oargs.good_nodes +'.tmp'
bak = oargs.good_nodes +'.bak'
with open(tmp, 'wt') as oFYaml:
yaml.dump(oGOOD_NODES, oFYaml)
LOG.info(f"Wrote {ilen} good relays to {oArgs.good_nodes}")
LOG.info(f"Wrote {ilen} good relays to {oargs.good_nodes}")
oFYaml.close()
if os.path.exists(oArgs.good_nodes):
os.rename(oArgs.good_nodes, bak)
os.rename(tmp, oArgs.good_nodes)
if os.path.exists(oargs.good_nodes):
os.rename(oargs.good_nodes, bak)
os.rename(tmp, oargs.good_nodes)
def iMain(lArgs):
global aTRUST_DB
@ -487,18 +478,18 @@ def iMain(lArgs):
global oGOOD_NODES
global lKNOWN_NODNS
parser = oMainArgparser()
oArgs = parser.parse_args(lArgs)
oargs = parser.parse_args(lArgs)
vsetup_logging(oArgs.log_level)
vsetup_logging(oargs.log_level)
if bAreWeConnected() is False:
raise SystemExit("we are not connected")
sFile = oArgs.torrc
sFile = oargs.torrc
if sFile and os.path.exists(sFile):
icheck_torrc(sFile, oArgs)
icheck_torrc(sFile, oargs)
twhitelist_set = set()
sFile = oArgs.good_contacts
sFile = oargs.good_contacts
if sFile and os.path.exists(sFile):
try:
with open(sFile, 'rt') as oFd:
@ -506,7 +497,7 @@ def iMain(lArgs):
LOG.info(f"{len(aTRUST_DB.keys())} trusted contacts from {sFile}")
# reverse lookup of fps to contacts
# but...
for k,v in aTRUST_DB.items():
for (k, v,) in aTRUST_DB.items():
if 'modified' not in v.keys():
v['modified'] = int(time.time())
aTRUST_DB_INDEX[k] = v
@ -520,19 +511,19 @@ def iMain(lArgs):
except Exception as e:
LOG.exception(f"Error reading YAML TrustDB {sFile} {e}")
if os.path.exists(oArgs.proxy_ctl):
controller = oMakeController(sSock=oArgs.proxy_ctl)
if os.path.exists(oargs.proxy_ctl):
controller = oGetStemController(log_level=oargs.log_level, sock_or_pair=oargs.proxy_ctl)
else:
port =int(oArgs.proxy_ctl)
controller = oMakeController(port=port)
port =int(oargs.proxy_ctl)
controller = oGetStemController(port=port)
vwait_for_controller(controller, oArgs.wait_boot)
vwait_for_controller(controller, oargs.wait_boot)
if oArgs.good_contacts:
good_contacts_tmp = oArgs.good_contacts + '.tmp'
if oargs.good_contacts:
good_contacts_tmp = oargs.good_contacts + '.tmp'
elt = controller.get_conf('UseMicrodescriptors')
if elt != '0' :
if elt != '0':
LOG.error('"UseMicrodescriptors 0" is required in your /etc/tor/torrc. Exiting.')
controller.set_conf('UseMicrodescriptors', 0)
# does it work dynamically?
@ -542,8 +533,8 @@ def iMain(lArgs):
if elt and elt != '{??}':
LOG.warn(f"{sEXCLUDE_EXIT_KEY} is in use already")
twhitelist_set.update(set(lYamlGoodNodes(oArgs.good_nodes)))
LOG.info(f"lYamlGoodNodes {len(twhitelist_set)} GuardNodes from {oArgs.good_nodes}")
twhitelist_set.update(set(lYamlGoodNodes(oargs.good_nodes)))
LOG.info(f"lYamlGoodNodes {len(twhitelist_set)} GuardNodes from {oargs.good_nodes}")
global oGOOD_NODES
t = set()
@ -556,23 +547,23 @@ def iMain(lArgs):
pass
if 'Onions' in oGOOD_NODES[oGOOD_ROOT].keys():
# Provides the descriptor for a hidden service. The **address** is the
# '.onion' address of the hidden service
# '.onion' address of the hidden service
w = set(oGOOD_NODES[oGOOD_ROOT]['Onions'])
if oArgs.white_onions:
w.update(oArgs.white_onions.split(','))
if oArgs.points_timeout > 0:
if oargs.white_onions:
w.update(oargs.white_onions.split(','))
if oargs.points_timeout > 0:
LOG.info(f"{len(w)} services will be checked from IntroductionPoints")
t.update(lIntroductionPoints(controller, w, itimeout=oArgs.points_timeout))
t.update(lIntroductionPoints(controller, w, itimeout=oargs.points_timeout))
if len(t) > 0:
LOG.info(f"IntroductionPoints {len(t)} relays from {len(w)} services")
twhitelist_set.update(t)
texclude_set = set()
if oArgs.bad_nodes and os.path.exists(oArgs.bad_nodes):
if False and oArgs.bad_sections:
if oargs.bad_nodes and os.path.exists(oargs.bad_nodes):
if False and oargs.bad_sections:
# BROKEN
sections = oArgs.bad_sections.split(',')
texclude_set = set(lYamlBadNodes(oArgs.bad_nodes,
sections = oargs.bad_sections.split(',')
texclude_set = set(lYamlBadNodes(oargs.bad_nodes,
lWanted=sections,
section=sEXCLUDE_EXIT_KEY))
LOG.info(f"Preloaded {len(texclude_set)} bad fps")
@ -583,7 +574,7 @@ def iMain(lArgs):
iTotalContacts = 0
aBadContacts = {}
lConds = oArgs.contact.split(',')
lConds = oargs.contact.split(',')
iR = 0
relays = controller.get_server_descriptors()
@ -612,17 +603,17 @@ def iMain(lArgs):
relay.contact = str(relay.contact, 'UTF-8')
if ('Empty' in lConds and not relay.contact) or \
('NoEmail' in lConds and relay.contact and not 'email:' in relay.contact):
('NoEmail' in lConds and relay.contact and 'email:' not in relay.contact):
texclude_set.add(relay.fingerprint)
continue
if not relay.contact or not 'ciissversion:' in relay.contact:
if not relay.contact or 'ciissversion:' not in relay.contact:
# should be unreached 'Empty' should always be in lConds
continue
iTotalContacts += 1
fp = relay.fingerprint
if relay.contact and not 'url:' in relay.contact:
if relay.contact and 'url:' not in relay.contact:
LOG.info(f"{fp} skipping bad contact - no url: {sofar}")
LOG.debug(f"{fp} {relay.contact} {sofar}")
texclude_set.add(fp)
@ -632,7 +623,7 @@ def iMain(lArgs):
# first rough cut
i = c.find('url:')
if i >=0:
c = c[i+4:]
c = c[i + 4:]
i = c.find(' ')
if i >=0: c = c[:i]
c = c.lstrip('https://').lstrip('http://').strip('/')
@ -682,15 +673,14 @@ def iMain(lArgs):
texclude_set.add(relay.fingerprint)
continue
b = aVerifyContact(list(a.values())[0],
relay.fingerprint,
oArgs.https_cafile,
timeout=oArgs.timeout,
host=oArgs.proxy_host,
port=oArgs.proxy_port)
if not b or not 'fps' in b or not b['fps'] or not b['url']:
oargs.https_cafile,
timeout=oargs.timeout,
host=oargs.proxy_host,
port=oargs.proxy_port)
# need to skip urllib3.exceptions.MaxRetryError
if not b or 'fps' not in b or not b['fps'] or not b['url']:
LOG.warn(f"{relay.fingerprint} did NOT VERIFY {sofar}")
LOG.debug(f"{relay.fingerprint} {b} {sofar}")
# If it's giving contact info that doesnt check out
@ -712,7 +702,7 @@ def iMain(lArgs):
aTRUST_DB[relay.fingerprint] = b
for elt in b['fps']:
aTRUST_DB_INDEX[elt] = b
if oArgs.good_contacts and oArgs.log_level <= 20:
if oargs.good_contacts and oargs.log_level <= 20:
# as we go along then clobber
with open(good_contacts_tmp, 'wt') as oFYaml:
yaml.dump(aTRUST_DB, oFYaml)
@ -724,38 +714,38 @@ def iMain(lArgs):
texclude_set = texclude_set.difference(tdns_urls)
LOG.info(f"{len(list(aTRUST_DB.keys()))} good contacts out of {iTotalContacts}")
if oArgs.torrc_output and texclude_set:
with open(oArgs.torrc_output, 'wt') as oFTorrc:
if oargs.torrc_output and texclude_set:
with open(oargs.torrc_output, 'wt') as oFTorrc:
oFTorrc.write(f"{sEXCLUDE_EXIT_KEY} {','.join(texclude_set)}\n")
oFTorrc.write(f"{sINCLUDE_EXIT_KEY} {','.join(aTRUST_DB_INDEX.keys())}\n")
oFTorrc.write(f"{sINCLUDE_GUARD_KEY} {','.join(oGOOD_NODES[oGOOD_ROOT]['GuardNodes'])}\n")
LOG.info(f"Wrote tor configuration to {oArgs.torrc_output}")
LOG.info(f"Wrote tor configuration to {oargs.torrc_output}")
oFTorrc.close()
if oArgs.bad_contacts and aBadContacts:
if oargs.bad_contacts and aBadContacts:
# for later analysis
with open(oArgs.bad_contacts, 'wt') as oFYaml:
with open(oargs.bad_contacts, 'wt') as oFYaml:
yaml.dump(aBadContacts, oFYaml)
oFYaml.close()
if oArgs.good_contacts != '' and aTRUST_DB:
if oargs.good_contacts != '' and aTRUST_DB:
with open(good_contacts_tmp, 'wt') as oFYaml:
yaml.dump(aTRUST_DB, oFYaml)
oFYaml.close()
if os.path.exists(oArgs.good_contacts):
bak = oArgs.good_contacts +'.bak'
os.rename(oArgs.good_contacts, bak)
os.rename(good_contacts_tmp, oArgs.good_contacts)
LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oArgs.good_contacts}")
if os.path.exists(oargs.good_contacts):
bak = oargs.good_contacts +'.bak'
os.rename(oargs.good_contacts, bak)
os.rename(good_contacts_tmp, oargs.good_contacts)
LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oargs.good_contacts}")
oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit'] = list(texclude_set)
oBAD_NODES[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS
vwrite_badnodes(oArgs, oBAD_NODES, str(len(texclude_set)))
vwrite_badnodes(oargs, oBAD_NODES, str(len(texclude_set)))
oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aTRUST_DB_INDEX.keys())
# GuardNodes are readonl
vwrite_goodnodes(oArgs, oGOOD_NODES, len(aTRUST_DB_INDEX.keys()))
vwrite_goodnodes(oargs, oGOOD_NODES, len(aTRUST_DB_INDEX.keys()))
retval = 0
try:
logging.getLogger('stem').setLevel(30)
@ -764,7 +754,7 @@ def iMain(lArgs):
LOG.info(f"{sEXCLUDE_EXIT_KEY} {len(texclude_set)} net bad exit relays")
controller.set_conf(sEXCLUDE_EXIT_KEY, texclude_set)
except stem.SocketClosed as e:
except stem.SocketClosed as e: # noqa
LOG.error(f"Failed setting {sEXCLUDE_EXIT_KEY} bad exit relays in Tor")
retval += 1
@ -772,7 +762,7 @@ def iMain(lArgs):
if aTRUST_DB_INDEX.keys():
LOG.info(f"{sINCLUDE_EXIT_KEY} {len(aTRUST_DB_INDEX.keys())} good relays")
controller.set_conf(sINCLUDE_EXIT_KEY, aTRUST_DB_INDEX.keys())
except stem.SocketClosed as e:
except stem.SocketClosed as e: # noqa
LOG.error(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
retval += 1
@ -783,12 +773,12 @@ def iMain(lArgs):
controller.set_conf(sINCLUDE_GUARD_KEY,
oGOOD_NODES[oGOOD_ROOT]['GuardNodes'])
cur = controller.get_conf('StrictNodes')
if oArgs.strict_nodes and int(cur) != oArgs.strict_nodes:
LOG.info(f"OVERRIDING StrictNodes to {oArgs.strict_nodes}")
controller.set_conf('StrictNodes', oArgs.strict_nodes)
if oargs.strict_nodes and int(cur) != oargs.strict_nodes:
LOG.info(f"OVERRIDING StrictNodes to {oargs.strict_nodes}")
controller.set_conf('StrictNodes', oargs.strict_nodes)
else:
LOG.info(f"StrictNodes is set to {cur}")
except stem.SocketClosed as e:
except stem.SocketClosed as e: # noqa
LOG.errro(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
retval += 1
@ -815,7 +805,7 @@ def iMain(lArgs):
controller.close()
except Exception as e:
LOG.warn(str(e))
sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n')
return retval