emdee/exclude_badExits
1
0
Fork 0
Code Issues Pull requests Releases Packages Wiki Activity

add exclude_badExits.txt

Browse source
This commit is contained in:
emdee 2022-11-29 14:52:48 +00:00
parent d08b34fd57
commit ec7c600d85
5 changed files with 248 additions and 173 deletions
Download patch file Download diff file Expand all files Collapse all files

148
exclude_badExits.py
View file

@ -17,7 +17,37 @@ or to relays that do not have ContactInfo that is verified to include them.
"""
__prolog__ = __doc__
__doc__ +="""But there's a problem, and your Tor notice.log will tell you about it:
sGOOD_NODES = """
---
GoodNodes:
EntryNodes: []
Relays:
# ExitNodes will be overwritten by this program
ExitNodes: []
IntroductionPoints: []
# use the Onions section to list onion services you want the
# Introduction Points whitelisted - these points may change daily
# Look in tor's notice.log for 'Every introduction point for service'
Onions: []
# use the Services list to list elays you want the whitelisted
# Look in tor's notice.log for 'Wanted to contact directory mirror'
Services: []
"""
sBAD_NODES = """
BadNodes:
# list the internet domains you know are bad so you don't
# waste time trying to download contacts from them.
ExcludeDomains: []
ExcludeNodes:
# BadExit will be overwritten by this program
BadExit: []
# list MyBadExit in --bad_sections if you want it used, to exclude nodes
# or any others as a list separated by comma(,)
MyBadExit: []
"""
__doc__ +=f"""But there's a problem, and your Tor notice.log will tell you about it:
you could exclude the relays needed to access hidden services or mirror
directories. So we need to add to the process the concept of a whitelist.
In addition, we may have our own blacklist of nodes we want to exclude,
@ -26,18 +56,16 @@ or use these lists for other applications like selektor.
So we make two files that are structured in YAML:
```
/etc/tor/yaml/torrc-goodnodes.yaml
GoodNodes:
Relays:
IntroductionPoints:
- NODEFINGERPRINT
...
{sGOOD_NODES}
By default all sections of the goodnodes.yaml are used as a whitelist.
Use the GoodNodes/Onions list to list onion services you want the
Introduction Points whitelisted - these points may change daily
Look in tor's notice.log for warnings of 'Every introduction point for service'
/etc/tor/yaml/torrc-badnodes.yaml
BadNodes:
ExcludeExitNodes:
BadExit:
- 0000000000000000000000000000000000000007
{sBAD_NODES}
```
That part requires [PyYAML](https://pyyaml.org/wiki/PyYAML)
https://github.com/yaml/pyyaml/ or ```ruamel```: do
@ -46,7 +74,7 @@ the advantage of the former is that it preserves comments.
(You may have to run this as the Tor user to get RW access to
/run/tor/control, in which case the directory for the YAML files must
be group Tor writeable, and its parents group Tor RX.)
be group Tor writeable, and its parent's directories group Tor RX.)
Because you don't want to exclude the introduction points to any onion
you want to connect to, ```--white_onions``` should whitelist the
@ -54,6 +82,13 @@ introduction points to a comma sep list of onions; we fixed stem to do this:
* https://github.com/torproject/stem/issues/96
* https://gitlab.torproject.org/legacy/trac/-/issues/25417
Use the GoodNodes/Onions list in goodnodes.yaml to list onion services
you want the Introduction Points whitelisted - these points may change daily.
Look in tor's notice.log for 'Every introduction point for service'
```notice_log``` will parse the notice log for warnings about relays and
services that will then be whitelisted.
```--torrc_output``` will write the torrc ExcludeNodes configuration to a file.
```--good_contacts``` will write the contact info as a ciiss dictionary
@ -78,7 +113,7 @@ list of fingerprints to ```ExitNodes```, a whitelist of relays to use as exits.
3. clean relays that don't have "good' contactinfo. (implies 1)
```=Empty,NoEmail,NotGood```
The default is ```=Empty,NotGood``` ; ```NoEmail``` is inherently imperfect
The default is ```Empty,NoEmail,NotGood``` ; ```NoEmail``` is inherently imperfect
in that many of the contact-as-an-email are obfuscated, but we try anyway.
To be "good" the ContactInfo must:
@ -87,7 +122,8 @@ To be "good" the ContactInfo must:
3. must support getting the file with a valid SSL cert from a recognized authority
4. (not in the spec but added by Python) must use a TLS SSL > v1
5. must have a fingerprint list in the file
6. must have the FP that got us the contactinfo in the fingerprint list in the file,
6. must have the FP that got us the contactinfo in the fingerprint list in the file.
For usage, do ```python3 exclude_badExits.py --help`
@ -175,28 +211,12 @@ sEXCLUDE_EXIT_GROUP = 'ExcludeNodes'
sINCLUDE_EXIT_KEY = 'ExitNodes'
oBAD_ROOT = 'BadNodes'
aBAD_NODES = safe_load("""
BadNodes:
ExcludeDomains: []
ExcludeNodes:
# BadExit will be overwritten
BadExit: []
# list MyBadExit in --bad_sections if you want it used
MyBadExit: []
""")
aBAD_NODES = safe_load(sBAD_NODES)
sGOOD_ROOT = 'GoodNodes'
sINCLUDE_GUARD_KEY = 'EntryNodes'
sEXCLUDE_DOMAINS = 'ExcludeDomains'
oGOOD_NODES = safe_load("""
GoodNodes:
EntryNodes: []
Relays:
ExitNodes: []
IntroductionPoints: []
Onions: []
Services: []
""")
aGOOD_NODES = safe_load(sGOOD_NODES)
lKNOWN_NODNS = []
tMAYBE_NODNS = set()
@ -230,13 +250,13 @@ def lYamlBadNodes(sFile,
return l
def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'):
global oGOOD_NODES
global aGOOD_NODES
l = []
if not yaml: return l
if os.path.exists(sFile):
with open(sFile, 'rt') as oFd:
o = safe_load(oFd)
oGOOD_NODES = o
aGOOD_NODES = o
if 'EntryNodes' in o[sGOOD_ROOT].keys():
l = o[sGOOD_ROOT]['EntryNodes']
# yq '.Nodes.IntroductionPoints|.[]' < /etc/tor/torrc-goodnodes.yaml
@ -644,9 +664,9 @@ def oMainArgparser(_=None):
default='127.0.0.1',
help='proxy host')
parser.add_argument('--proxy_port', '--proxy-port', default=9050, type=int,
help='proxy control port')
help='proxy socks port')
parser.add_argument('--proxy_ctl', '--proxy-ctl',
default='/run/tor/control' if os.path.exists('/run/tor/control') else 9051,
default='/run/tor/control' if os.path.exists('/run/tor/control') else '9051',
type=str,
help='control socket - or port')
@ -689,9 +709,12 @@ def oMainArgparser(_=None):
parser.add_argument('--torrc_output', type=str,
default=os.path.join(ETC_DIR, 'torrc.new'),
help="Write the torrc configuration to a file")
parser.add_argument('--hs_dir', type=str,
default='/var/lib/tor',
help="Parse the files name hostname below this dir to find Hidden Services to whitelist")
parser.add_argument('--notice_log', type=str,
default='',
help="Parse the notice log for relays and services (not yet)")
help="Parse the notice log for relays and services")
parser.add_argument('--relays_output', type=str,
default=os.path.join(ETC_DIR, 'relays.json'),
help="Write the download relays in json to a file")
@ -718,23 +741,23 @@ def vwrite_good_contacts(oargs):
yaml.dump(aBAD_CONTACTS_DB, oFYaml)
oFYaml.close()
def vwrite_badnodes(oargs, aBAD_NODES, slen):
def vwrite_badnodes(oargs, aBAD_NODES, slen, stag):
if not aBAD_NODES: return
tmp = oargs.bad_nodes +'.tmp'
bak = oargs.bad_nodes +'.bak'
with open(tmp, 'wt') as oFYaml:
yaml.dump(aBAD_NODES, oFYaml)
LOG.info(f"Wrote {slen} to {oargs.bad_nodes}")
LOG.info(f"Wrote {slen} to {stag} in {oargs.bad_nodes}")
oFYaml.close()
if os.path.exists(oargs.bad_nodes):
os.rename(oargs.bad_nodes, bak)
os.rename(tmp, oargs.bad_nodes)
def vwrite_goodnodes(oargs, oGOOD_NODES, ilen):
def vwrite_goodnodes(oargs, aGOOD_NODES, ilen):
tmp = oargs.good_nodes +'.tmp'
bak = oargs.good_nodes +'.bak'
with open(tmp, 'wt') as oFYaml:
yaml.dump(oGOOD_NODES, oFYaml)
yaml.dump(aGOOD_NODES, oFYaml)
LOG.info(f"Wrote {ilen} good relays to {oargs.good_nodes}")
oFYaml.close()
if os.path.exists(oargs.good_nodes):
@ -1022,23 +1045,31 @@ def tWhitelistSet(oargs, controller):
LOG.info(f"lYamlGoodNodes {len(twhitelist_set)} EntryNodes from {oargs.good_nodes}")
t = set()
if sGOOD_ROOT in oGOOD_NODES and 'Relays' in oGOOD_NODES[sGOOD_ROOT] and \
'IntroductionPoints' in oGOOD_NODES[sGOOD_ROOT]['Relays'].keys():
t = set(oGOOD_NODES[sGOOD_ROOT]['Relays']['IntroductionPoints'])
if 'IntroductionPoints' in aGOOD_NODES[sGOOD_ROOT]['Relays'].keys():
t = set(aGOOD_NODES[sGOOD_ROOT]['Relays']['IntroductionPoints'])
if oargs.hs_dir and os.path.exists(oargs.hs_dir):
for (dirpath, dirnames, filenames,) in os.walk(oargs.hs_dir):
for f in filenames:
if f != 'hostname': continue
with open(os.path.join(dirpath, f), 'rt') as oFd:
son = oFd.read()
t.update(son)
LOG.info(f"Added {son} to the list for Introduction Points")
if oargs.notice_log and os.path.exists(oargs.notice_log):
tmp = tempfile.mktemp()
i = os.system(f"grep 'Every introduction point for service' {oargs.notice_log} |sed -e 's/.* service //' -e 's/ is .*//'|sort -u |sed -e '/ /d' > {tmp}")
if i:
with open(tmp, 'rt') as oFd:
lnew = oFd.readlines()
t.update(set(lnew))
tnew = {elt.strip() for elt in oFd.readlines()}
t.update(tnew)
LOG.info(f"Whitelist {len(lnew)} services from {oargs.notice_log}")
os.remove(tmp)
w = set()
if sGOOD_ROOT in oGOOD_NODES and 'Services' in oGOOD_NODES[sGOOD_ROOT].keys():
w = set(oGOOD_NODES[sGOOD_ROOT]['Services'])
if sGOOD_ROOT in aGOOD_NODES and 'Services' in aGOOD_NODES[sGOOD_ROOT].keys():
w = set(aGOOD_NODES[sGOOD_ROOT]['Services'])
if len(w) > 0:
LOG.info(f"Whitelist {len(w)} relays from {sGOOD_ROOT}/Services")
@ -1054,10 +1085,10 @@ def tWhitelistSet(oargs, controller):
twhitelist_set.update(w)
w = set()
if 'Onions' in oGOOD_NODES[sGOOD_ROOT].keys():
if 'Onions' in aGOOD_NODES[sGOOD_ROOT].keys():
# Provides the descriptor for a hidden service. The **address** is the
# '.onion' address of the hidden service
w = set(oGOOD_NODES[sGOOD_ROOT]['Onions'])
w = set(aGOOD_NODES[sGOOD_ROOT]['Onions'])
if oargs.white_onions:
w.update(oargs.white_onions.split(','))
if oargs.points_timeout > 0:
@ -1088,7 +1119,7 @@ def iMain(lArgs):
global aGOOD_CONTACTS_FPS
global aBAD_CONTACTS_DB
global aBAD_NODES
global oGOOD_NODES
global aGOOD_NODES
global lKNOWN_NODNS
global aRELAYS_DB
global aRELAYS_DB_INDEX
@ -1198,7 +1229,7 @@ def iMain(lArgs):
with open(oargs.torrc_output, 'wt') as oFTorrc:
oFTorrc.write(f"{sEXCLUDE_EXIT_GROUP} {','.join(texclude_set)}\n")
oFTorrc.write(f"{sINCLUDE_EXIT_KEY} {','.join(aGOOD_CONTACTS_FPS.keys())}\n")
oFTorrc.write(f"{sINCLUDE_GUARD_KEY} {','.join(oGOOD_NODES[sGOOD_ROOT]['EntryNodes'])}\n")
oFTorrc.write(f"{sINCLUDE_GUARD_KEY} {','.join(aGOOD_NODES[sGOOD_ROOT]['EntryNodes'])}\n")
LOG.info(f"Wrote tor configuration to {oargs.torrc_output}")
oFTorrc.close()
@ -1214,12 +1245,13 @@ def iMain(lArgs):
aBAD_NODES[oBAD_ROOT][sEXCLUDE_EXIT_GROUP]['BadExit'] = list(texclude_set)
aBAD_NODES[oBAD_ROOT][sEXCLUDE_DOMAINS] = lKNOWN_NODNS
if oargs.bad_nodes:
vwrite_badnodes(oargs, aBAD_NODES, str(len(texclude_set)))
stag = sEXCLUDE_EXIT_GROUP + '/BadExit'
vwrite_badnodes(oargs, aBAD_NODES, str(len(texclude_set)), stag)
oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aGOOD_CONTACTS_FPS.keys())
aGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aGOOD_CONTACTS_FPS.keys())
# EntryNodes are readony
if oargs.good_nodes:
vwrite_goodnodes(oargs, oGOOD_NODES, len(aGOOD_CONTACTS_FPS.keys()))
vwrite_goodnodes(oargs, aGOOD_NODES, len(aGOOD_CONTACTS_FPS.keys()))
vwritefinale(oargs)
@ -1245,15 +1277,15 @@ def iMain(lArgs):
LOG.debug(repr(l))
retval += 1
if 'EntryNodes' in oGOOD_NODES[sGOOD_ROOT].keys():
if 'EntryNodes' in aGOOD_NODES[sGOOD_ROOT].keys():
try:
LOG.info(f"{sINCLUDE_GUARD_KEY} {len(oGOOD_NODES[sGOOD_ROOT]['EntryNodes'])} guard nodes")
LOG.info(f"{sINCLUDE_GUARD_KEY} {len(aGOOD_NODES[sGOOD_ROOT]['EntryNodes'])} guard nodes")
# FixMe for now override StrictNodes it may be unusable otherwise
controller.set_conf(sINCLUDE_GUARD_KEY,
oGOOD_NODES[sGOOD_ROOT]['EntryNodes'])
aGOOD_NODES[sGOOD_ROOT]['EntryNodes'])
except (Exception, stem.InvalidRequest, stem.SocketClosed,) as e: # noqa
LOG.error(f"Failed setting {sINCLUDE_GUARD_KEY} guard nodes in Tor {e}")
LOG.debug(repr(list(oGOOD_NODES[sGOOD_ROOT]['EntryNodes'])))
LOG.debug(repr(list(aGOOD_NODES[sGOOD_ROOT]['EntryNodes'])))
retval += 1
cur = controller.get_conf('StrictNodes')