From 0f0f540c82a0859cdd785e315bf7fff109cf6dff Mon Sep 17 00:00:00 2001 From: Kufat Date: Sat, 30 Jan 2021 19:15:54 -0500 Subject: [PATCH 01/11] Initial commit for email/IP scoring and database storage. Tested to load in sopel but not on a server. --- sopel/modules/emailcheck.py | 282 +++++++++++++++++++++ sopel/modules/ip.py | 489 ++++++++++++++++++++++++++++++------ 2 files changed, 697 insertions(+), 74 deletions(-) create mode 100644 sopel/modules/emailcheck.py diff --git a/sopel/modules/emailcheck.py b/sopel/modules/emailcheck.py new file mode 100644 index 0000000000..6bb5e9d80c --- /dev/null +++ b/sopel/modules/emailcheck.py @@ -0,0 +1,282 @@ +# coding=utf-8 +""" +emailcheck.py - Watch oper messages for new nicks being registered +Copyright © 2021, Kufat +Based on existing sopel code. +Licensed under the Eiffel Forum License 2. +""" + +import logging +import re +import urllib + +import sqlalchemy.sql + +from dataclasses import dataclass + +from sopel import db, module +from sopel.config.types import FilenameAttribute, StaticSection, ValidatedAttribute, ListAttribute +from sopel.tools import events, target, Identifier + +from sqlalchemy import Column, String, Float, Boolean, TIMESTAMP +from sqlalchemy.ext.declarative import declarative_base + +try: + from ip import get_exemption +except: + def get_exemption(ip): + return "Can't access exemptions; failing safe" + +EMAIL_REGEX = re.compile(r"([a-zA-Z0-9_.+-]+)@([a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)") +IRCCLOUD_USER_REGEX = re.compile(r"[us]id[\d]{4,}") +DOMAIN_LEN = 50 + +DEFAULT_EXEMPT_SUFFIXES = { + "@gmail.com", + "@hotmail.com", + "@protonmail.com", + ".edu" +} + +KILL_STR = ":Use of disposable email service for nick registration" + +LOGGER = logging.getLogger(__name__) + +BASE = declarative_base() + +#SQLAlchemy container class +class KnownEmails(BASE): + __tablename__ = 'known_emails' + domain = Column(String(DOMAIN_LEN), primary_key=True) + first_nick = Column(String(40)) + score = Column(Float) + flag_disposable = Column(Boolean) + flag_recent_abuse = Column(Boolean) + first_seen = Column(TIMESTAMP, server_default=sqlalchemy.sql.func.now()) + +class EmailCheckSection(StaticSection): + IPQS_key = ValidatedAttribute('IPQS_key') + disallow_threshold = ValidatedAttribute("disallow_threshold", parse=float) + malicious_threshold = ValidatedAttribute("malicious_threshold", parse=float) + gline_time = ValidatedAttribute('gline_time') + #TODO; just hard-coded ones for now + exempt_suffixes = ListAttribute("exempt_suffixes") + warn_chans = ListAttribute("warn_chans") + protect_chans = ListAttribute("protect_chans") + +def configure(config): + config.define_section('emailcheck', EmailCheckSection) + config.emailcheck.configure_setting('IPQS_key', + 'Access key for IPQS service') + config.emailcheck.configure_setting('disallow_threshold', + 'Addresses with scores >= this will be disallowed; no punishment', + default=50.0) + config.emailcheck.configure_setting('malicious_threshold', + 'Addresses with scores >= this will be interpreted as attacks', + default=75.0) + config.emailcheck.configure_setting('gline_time', + 'Users attempting to register with malicious addresses will be ' + 'glined for this priod of time.', + default="24h") + config.emailcheck.configure_setting('exempt_suffixes', + 'Suffixes (TLD, whole domain, etc.) to exempt from checking') + config.emailcheck.configure_setting('warn_chans', + 'List of channels to warn when a suspicious user is detected. ' + 'May be empty.') + config.emailcheck.configure_setting('protect_chans', + 'List of channels to +R after malicious attempt to reg. ' + 'May be empty.') + +def setup(bot): + bot.config.define_section('emailcheck', EmailCheckSection) + +@dataclass +class Email: + user: str + domain: str + def get_address(self): + return f'{self.user}@{self.domain}' + def __str__(self): + return self.get_address() + def __post_init__(self): + self.domain = self.domain.lower() + +@dataclass +class DomainInfo: + score: float + flag_disposable: bool + flag_recent_abuse: bool + +def alert(bot, alert_msg: str, log_err: bool = False): + for channel in config.emailcheck.warn_chans: + bot.say(alert_msg, channel) + if log_err: + LOGGER.error(alert_msg) + +def add_badmail(bot, email): + #Right now we're BADMAILing whole domains. This might change. + bot.write("NICKSERV", "badmail", "add", f'*@{email.domain}') + +def fdrop(bot, nick: str): + bot.write("NICKSERV", "fdrop", nick.lower()) + +def gline_ip(bot, ip: str, duration: str): + bot.write("GLINE", f'*@{ip}', duration, KILL_STR) + +def gline_username(bot, nick: str, duration: str): + if known_user := bot.users.get(Identifier(nick)): + username = known_user.user.lower() # Should already be lowercase + if IRCCLOUD_USER_REGEX.match(username): + bot.write("GLINE", f'{username}@*', duration, KILL_STR) + return + else: + alert(bot, f"User {nick} had unexpected non-IRCCloud username {username}", true) + else: + alert(bot, f"Couldn't find irccloud uid/sid for {nick} to G-line!", true) + kill_nick(bot, nick) # Something went wrong with G-line, so fall back to /kill + +def kill_nick(bot, nick: str): + bot.write("KILL", nick.lower(), KILL_STR) + +def gline_strategy(bot, nick): + if (known_user := bot.users.get(Identifier(nick))): + if hasattr(known_user, "ip"): + ip = known_user.ip + exemption = get_exemption(ip) + if exemption: + if "irccloud" in exemption.lower(): + # IRCCloud special case: ban uid/sid + return ["gline_username", known_user.user] + else: # Fully exempt, so no g-line + return None + else: # No exemption + return ["gline_ip", ip] + else: # Fail safely + return None + +def gline_or_kill(bot, nick: str, duration: str): + if strategy := gline_strategy(bot, nick): + if strategy[0] == "gline_ip": + gline_ip(bot, strategy[1], duration) + elif strategy[0] == "gline_username": + gline_username(bot, strategy[1], duration) + else: + alert(bot, f"Unknown strategy {strategy} for nick {nick}", true) + kill_nick(bot, nick) # safest option + else: + kill_nick(bot, nick) # duration ignored + +def protect_chans(bot): + for chan in config.emailcheck.protect_chans: + bot.write("MODE", chan, "+R") + +def malicious_response(bot, nick: str, email): + fdrop(bot, nick) + add_badmail(bot, email) + say(f"You have been temporarily banned from this network because {email.domain} " + "has a history of spam or abuse, and/or is a disposable email domain. " + "If this is a legitimate domain, contact staff for assistance.", + nick.lower()) + gline_or_kill(bot, nick, config.emailcheck.gline_time) + protect_chans(bot) + alert(bot, f"ALERT: User {nick} attempted to register a nick with disposable/spam domain {email.domain}!") + +def disallow_response(bot, nick: str, email): + fdrop(bot, nick) + add_badmail(bot, email) + say(f"Your registration has been disallowed because {email.domain} appears to be suspicious. " + "If this is a legitimate domain, contact staff for assistance.", + nick.lower()) + alert(bot, f"WARNING: User {nick} attempted to register a nick with suspicious domain {email.domain}.") + +def fetch_IPQS_email_score( + email_addr: str, + key: str, + fast: bool = True + ) -> tuple[float, bool, bool]: #score, disposable, has recent abuse flag set + '''Perform lookup on a specific email adress using ipqualityscore.com''' + email_str = urllib.parse.quote(email_addr) + faststr = str(bool(fast)).lower() #lower + handle None and other garbage + params = urllib.parse.urlencode({'fast': faststr}) + with urllib.request.urlopen( + f"https://ipqualityscore.com/api/json/email/{key}/{email_str}?{params}") as url: + data = json.loads(url.read().decode()) + LOGGER.debug(data) + if not data['success']: + errstr = f"{email_addr} lookup failed with {data['message']}" + LOGGER.error(errstr) + raise RuntimeError(errstr) + return (data['fraud_score'], data["disposable"], data["recent_abuse"]) + +def get_email_score_from_db(session, email): + query_result = session.query(KnownEmails)\ + .filter(KnownEmails.domain == email.domain)\ + .one_or_none() + if query_result: + #Any known problematic provider should've been BADMAILed by now, but... + return DomainInfo(query_result.score, + query_result.flag_disposable, + query_result.flag_recent_abuse) + +def store_email_score_in_db(session, email, nick, IPQSresult): + new_known_email = KnownEmails(domain= email.doman[:DOMAIN_LEN], + first_nick= nick, + score= IPQSresult[0], + flag_disposable= IPQSresult[1], + flag_recent_abuse= IPQSresult[2]) + session.add(new_known_email) + session.commit() + +def retrieve_score(bot, email, nick): + session = bot.db.ssession() + try: + if retval := get_email_score_from_db(session, email): + return retval + else: + if IPQSresult := fetch_IPQS_email_score(email, config.emailcheck.IPQS_key): + store_email_score_in_db(session, email, nick, IPQSresult) + return IPQSresult + else: #Shouldn't be possible + raise RuntimeError("Couldn't retrieve IPQS!") + except SQLAlchemyError: + session.rollback() + raise + finally: + session.remove() + +def check_email(bot, email, nick): + if any(map(email.endswith, DEFAULT_EXEMPT_SUFFIXES)): + #email is exempt + LOGGER.info(f'Email {email} used by {nick} is on the exemption list.') + return None # No lookup, no result + #Check database + else: + return retrieve_score(bot, email, nick) + +# ExampleAccount REGISTER: ExampleNick to foo@example.com +# (note the 0x02 bold chars) +@module.rule(r'(\S*)\s*REGISTER: ?([\S]+?)? to ?(\S+)@(\S+?)?$') +@module.event("PRIVMSG") +@module.priority("high") +def handle_ns_register(bot, trigger): + if "nickserv" != trigger.sender.lower(): + LOGGER.warning(f"Fake registration notice from {trigger.sender.lower()}!") + return + #It's really from nickserv. + _, nick, email_user, email_domain = trigger.groups() + email = Email(email_user, email_domain) + try: + if res := check_email(bot, email_user, email_domain, nick): #may be None, in which case we're done + if res.flag_disposable or ( + res.score >= config.emailcheck.malicious_threshold): + malicious_response(bot, nick, email) + elif res.flag_recent_abuse or ( + res.score >= config.emailcheck.disallow_threshold): + disallow_response(bot, nick, email) + else: + #already logged server response + return LOGGER.debug(f'Registration of {nick} to {email} OK.') + except: + alert(f"Lookup for f{nick} with email @f{domain} failed! Keep an eye on them.") + + diff --git a/sopel/modules/ip.py b/sopel/modules/ip.py index d2c2786008..1569f348f5 100644 --- a/sopel/modules/ip.py +++ b/sopel/modules/ip.py @@ -10,52 +10,227 @@ from __future__ import unicode_literals, absolute_import, print_function, division +import ipaddress import logging import os import socket import tarfile +import typing import geoip2.database - -from sopel.config.types import FilenameAttribute, StaticSection -from sopel.module import commands, example -from sopel.tools import web - -urlretrieve = None -try: - from urllib import urlretrieve -except ImportError: - try: - # urlretrieve has been put under urllib.request in Python 3. - # It's also deprecated so this should probably be replaced with - # urllib2. - from urllib.request import urlretrieve - except ImportError: - pass - +import re +import urllib.request, json + +import sqlalchemy.sql + +from random import randint +from urllib.request import urlretrieve + +#from minfraud import Client + +from sopel import module +from sopel.config.types import FilenameAttribute, StaticSection, ValidatedAttribute, ListAttribute +from sopel.tools import web, events, target, Identifier + +from sqlalchemy import Column, Integer, String, Float, Boolean, TIMESTAMP, Text +from sqlalchemy.ext.declarative import declarative_base + +IRCCLOUD_IP = [ + "2001:67c:2f08::/48", + "2a03:5180:f::/64", + "5.254.36.56/29", + "192.184.8.73", + "192.184.8.103", + "192.184.9.108", + "192.184.9.110", + "192.184.9.112", + "192.184.10.9", + "192.184.10.118", + ] + +IRCCLOUD_REASON = "IRCCloud" + +MIBBIT_IP = [ + "207.192.75.252", + "64.62.228.82", + "78.129.202.38", + "109.169.29.95" + ] + +MIBBIT_REASON = "Mibbit" + +#Hardcoded for safety +EXEMPT_IP = [ + ("104.248.43.234", "Lazar, DigitalOcean"), + ("13.59.180.136", "War_ Limnoria bot"), + ("138.68.23.34", "Approved bot (Idlebot, aismallard)"), + ("18.132.171.104", "TARS"), + ("3.136.223.150", "bluesoul"), + ("54.174.11.206", "Helen"), + ("69.115.75.7", "Kufat and bots"), + ("94.159.196.226", "docazra, longtime user, IP is on dnsbl"), + ("2604:a880:2:d0::250:9001", "Approved bot (Idlebot, aismallard)"), + ("91.132.86.177", "bluesoul's bouncer"), + ("2001:470:1f07:13b::/64", "kufat's tunnelbroker"), + ] LOGGER = logging.getLogger(__name__) - +who_reqs = {} # Keeps track of reqs coming from this plugin, rather than others + +# This dict will mostly be used as a list (walking each element) but having it as a dict +# is useful for preventing duplicate entries. +exemptions = {ipaddress.ip_network(i):reason for i, reason in EXEMPT_IP} +exemptions.update(( (ipaddress.ip_network(i), IRCCLOUD_REASON) for i in IRCCLOUD_IP)) +exemptions.update(( (ipaddress.ip_network(i), MIBBIT_REASON) for i in MIBBIT_IP)) + +BASE = declarative_base() + +# This table will only receive inserts, not updates. +class KnownIPs(BASE): + __tablename__ = 'known_ips' + ip = Column(String(50), primary_key=True, unique=True, index=True) + score = Column(Float) + flag_recent_abuse = Column(Boolean) + flag_is_proxy = Column(Boolean) + insert_time = Column(TIMESTAMP, server_default=sqlalchemy.sql.func.now()) + +class ExemptIPs(BASE): + __tablename__ = 'exempt_ips' + ip = Column(String(50), primary_key=True, unique=True, index=True) + # Define type of exemption. + # GLine username e.g. uid123@* if true, no g-line at all if false. For IRCCloud. + gline_username = Column(Boolean) + exempt_reason = Column(Text) + +# Existing rows in this table will be updated when a user is seen +# from the same nick/IP combination. +# TODO Deferred feature. +class KnownUsers(BASE): + __tablename__ = 'known_users' + nick = Column(String(40), primary_key=True) + ip = Column(String(50), primary_key=True) + cloaked_host = Column(String(40)) + last_seen = Column(TIMESTAMP, server_default=sqlalchemy.sql.func.now()) class GeoipSection(StaticSection): GeoIP_db_path = FilenameAttribute('GeoIP_db_path', directory=True) """Path of the directory containing the GeoIP database files.""" - + IPQS_key = ValidatedAttribute('IPQS_key') + warn_threshold = ValidatedAttribute('warn_threshold', parse=float) + malicious_threshold = ValidatedAttribute('malicious_threshold', parse=float) + warn_chans = ListAttribute("warn_chans") + protect_chans = ListAttribute("protect_chans") def configure(config): - """ - | name | example | purpose | - | ---- | ------- | ------- | - | GeoIP\\_db\\_path | /home/sopel/GeoIP/ | Path to the GeoIP database files | - """ config.define_section('ip', GeoipSection) config.ip.configure_setting('GeoIP_db_path', 'Path of the GeoIP db files') + config.ip.configure_setting('IPQS_key', + 'Access key for IPQS service') + config.ip.configure_setting('warn_threshold', + 'Addresses with scores >= this will generate an alert', + default=50.0) + config.ip.configure_setting('malicious_threshold', + 'Addresses with scores >= this will be z-lined', + default=70.0) + config.ip.configure_setting('warn_chans', + 'List of channels to warn when a suspicious user is detected. ' + 'May be empty.') + config.ip.configure_setting('protect_chans', + 'List of channels to +R after malicious attempt to reg. ' + 'May be empty.') def setup(bot): bot.config.define_section('ip', GeoipSection) +def alert(bot, alert_msg: str, log_err = False): + for channel in config.ip.warn_chans: + bot.say(alert_msg, channel) + if log_err: + LOGGER.error(alert_msg) + +def get_exemption(host): + if isinstance(host, ip.IPv4Address) or isinstance(host, ip.IPv6Address): + host = ip + else: + try: + ip = ipaddress.ip_address(socket.getaddrinfo(host, None)[0][4][0]) + except: + raise + for network, reason in exemptions.items(): + if ip in network: + return reason + return None + +def fetch_IPQS_score( + ip_addr: typing.Union[ipaddress.IPv4Address, ipaddress.IPv6Address], + key: str, + allow_public_access_points: bool = True, + strictness: int = 1, + fast: bool = False, + mobile: bool = False + ) -> tuple[float, bool, bool]: #score, is proxy, has recent abuse flag set + '''Perform lookup on a specific IP adress using ipqualityscore.com''' + params = urllib.parse.urlencode({ + # Case to lower + handle None and other garbage + 'allow_public_access_points': "true" if allow_public_access_points else "false", + 'strictness': int(strictness), + 'fast': "true" if fast else "false", + 'mobile': "true" if mobile else "false", + }) + # ip_addr sourced from server, not user, so sanitization already done + with urllib.request.urlopen( + f"https://ipqualityscore.com/api/json/ip/{key}/{str(ip_addr)}?{params}") as url: + data = json.loads(url.read().decode()) + LOGGER.info(data) + if not data['success']: + errstr = f"{ip_addr} lookup failed with {data['message']}" + LOGGER.error(errstr) + raise RuntimeError(errstr) + return (data['score'], data["proxy"], data["recent_abuse"]) + +def get_ip_score_from_db(session, ip): + query_result = session.query(KnownIPs)\ + .filter(KnownIPs.ip == str(ip))\ + .one_or_none() + if query_result: + #Any known problematic provider should've been BADMAILed by now, but... + return (query_result.score, + query_result.flag_recent_abuse, + query_result.flag_proxy + ) + +def store_ip_score_in_db(session, ip, nick, IPQSresult): + new_known_ip = KnownEmails(ip= ip, + score= IPQSresult[0], + flag_recent_abuse= IPQSresult[1], + flag_is_proxy= IPQSresult[2]) + session.add(new_known_ip) + session.commit() + +def retrieve_score(bot, ip, nick, do_fetch = True): + session = bot.db.ssession() + try: + if retval := get_ip_score_from_db(session, ip): + return retval + elif do_fetch: + if IPQSresult := fetch_IPQS_ip_score(ip, config.emailcheck.IPQS_key): + store_ip_score_in_db(session, ip, nick, IPQSresult) + return IPQSresult + else: #Shouldn't be possible + raise RuntimeError("Couldn't retrieve IPQS!") + else: + # If do_fetch is false, this is a best-effort request and shouldn't use up a query + return None + except SQLAlchemyError: + session.rollback() + raise + finally: + session.remove() + +def _add_exemption(ip, reason): + exemptions[ipaddress.ip_network(ip)] = reason def _decompress(source, target, delete_after_decompression=True): """Decompress just the database from the archive""" @@ -112,80 +287,246 @@ def _find_geoip_db(bot): else: return False +def populate_user(bot, user, ip, host, nick): + LOGGER.debug('Adding: %s!%s@%s with IP %s', nick, user, host, ip) + + user = bot.users.get(nick) or target.User(nick, user, host) + if ip: + user.ip = ip # Add nonstandard field + bot.users[nick] = user # no-op if user was in users, needed otherwise + +def examine_user(bot, user, ip, host, nick): + populate_user(bot, user, ip, host, nick) + res = retrieve_score(bot, ip, nick) + if res: + score, is_proxy, is_recent_abuse = res + if( is_prox or is_recent_abuse or score >= config.ip.malicious_threshold ): + alert(bot, f"Ops: Nick {nick} has abuse score {score}, proxy: {is_prox}, " + "recent_abuse: {is_recent_abuse}; z-lining!") + bot.write("ZLINE", ip, "24h", f":Auto z-line {nick}.") + protect_chans(bot) + elif score >= config.ip.warn_threshold: + alert(bot, f"Ops: Nick {nick} has abuse score {score}; keep an eye on them.") + return res + +@module.event(events.RPL_WHOSPCRPL) +@module.priority('high') +def recv_whox_ip(bot, trigger): + """Track ``WHO`` responses when ``WHOX`` is enabled.""" + #LOGGER.debug('Receiving who: %s', trigger.args[1]) + if len(trigger.args) < 2 or trigger.args[1] not in who_reqs: + # Ignored, some other plugin probably called WHO + return + #it's us + # :safe.oh.us.irc.scpwiki.com 354 Kufat 0 kufat 2001:470:1f07:13b::1 gatekeeper.kufat.net :Kufat + if len(trigger.args) != 6: + return LOGGER.warning('While populating the IP DB, a WHO response was malformed.') + _, _, user, ip, host, nick = trigger.args + examine_user(bot, user, ip, host, nick) + +@module.event(events.RPL_ENDOFWHO) +@module.priority('high') +def end_who_ip(bot, trigger): + """Handle the end of a response to a ``WHO`` command (if needed).""" + if 'WHOX' in bot.isupport: + who_reqs.pop(trigger.args[1], None) + +@module.event(events.RPL_YOUREOPER) +@module.priority('high') +def send_who(bot, _): + if 'WHOX' in bot.isupport: + # WHOX syntax, see http://faerion.sourceforge.net/doc/irc/whox.var + # Needed for accounts in WHO replies. The random integer is a param + # to identify the reply as one from this command, because if someone + # else sent it, we have no way to know what the format is. + + # 'x' indicates uncloaked address. This is triggered by + # RPL_YOUREOPER because that functionality is restricted to opers. + rand = str(randint(0, 999)) + while rand in who_reqs: + rand = str(randint(0, 999)) + LOGGER.debug('Sending who: %s', rand) + who_reqs[rand] = True + bot.write([r'WHO * n%nuhti,' + rand]) + +#:safe.oh.us.irc.scpwiki.com NOTICE Kufat :*** CONNECT: Client connecting on port 6697 (class main): ASNbot!sopel@ool-45734b07.dyn.optonline.net (69.115.75.7) [Sopel: https://sopel.chat/] +@module.rule(r'.*Client connecting .*: (\S*)!(\S*)@(\S*) \((.*)\)') +@module.event("NOTICE") +@module.priority("high") +def handle_snotice_conn(bot, trigger): + LOGGER.debug("Saw connect line: [%s] from [%s]", trigger.raw, trigger.sender) + #Only servers may have '.' in the sender name, so this isn't spoofable + if "scpwiki.com" in trigger.sender: + nick, user, host, ip = trigger.groups() + examine_user(bot, user, ip, None, nick) #cloaked host not known + #Be **certain** we don't waste our lookups on irccloud + if any(host.endswith(s) for s in (".irccloud.com", ".mibbit.com")): + return + # We need to check if the IP is in any exempt CIDR ranges + if get_exemption(ip): + return + + res = examine_user(bot, user, ip, host, nick) + if res: + # Acted on above; just log here + score, is_proxy, is_recent_abuse = res + LOGGER.debug(f"handle_snotice_conn: {nick}!{user}@{host} ({ip}) had " + "score {score}, proxy: {is_prox}, " + "recent_abuse: {is_recent_abuse}") + +# NICK: User Kufat-bar changed their nickname to Kufat-foo +# This is redundant for users the bot can see in-channel but needed for users with no common channel +@module.rule(r'.*User (\S+) changed their nickname to (\S+).*') +@module.event("NOTICE") +@module.priority("high") +def handle_snotice_ren(bot, trigger): + LOGGER.debug("Saw nick change line: [%s] from [%s]", trigger.raw, trigger.sender) + if "scpwiki.com" in trigger.sender: + oldnick = Identifier(trigger.group(1)) + newnick = Identifier(trigger.group(2)) + if olduser := bot.users.get(oldnick): + populate_user(bot, olduser.user, olduser.ip, olduser.host, newnick) + +@module.require_privilege(module.OP) +@module.commands('ip_exempt') +@module.example('.ip_exempt 8.8.8.8') +def ip_exempt(bot, trigger): + if not trigger.group(3): + return bot.reply("You must specify an IP or range in CIDR format to exempt.") + elif not trigger.group(4): + return bot.reply("You must specify a reason for the exemption.") + ipstr = trigger.group(3) + if '*' in ipstr: + return bot.reply("Use CIDR format (1.2.3.0/24) rather than wildcard format (1.2.3.*)") + reason = trigger.group(2).lstrip(ipstr).lstrip() + try: + _add_exemption(ipstr, reason) + except ValueError as e: + return bot.reply(f"Could not add exemption for {ipstr} because: {str(e)}") -@commands('iplookup', 'ip') -@example('.ip 8.8.8.8', +@module.require_privilege(module.HALFOP) +@module.commands('iplookup', 'ip') +@module.example('.ip 8.8.8.8', r'\[IP\/Host Lookup\] Hostname: \S*dns\S*\.google\S*( \| .+?: .+?)+ \| ISP: AS15169 \S+', re=True, ignore='Downloading GeoIP database, please wait...', online=True) def ip(bot, trigger): + if trigger.is_privmsg and ( trigger.account is None or trigger.account.lower() != "kufat" ): + return + full = ( ( trigger.sender.lower() in ("#skipirc-staff", "#kufat") ) or + ( trigger.is_privmsg and trigger.account.lower() == "kufat" ) ) + irccloud = False + mibbit = False + nick = None """IP Lookup tool""" # Check if there is input at all if not trigger.group(2): - return bot.reply("No search term.") + return bot.reply("Usage: '.ip (Nick or address) [lookup]'. " + "If 'lookup' is specified, will look up IP score if not known.") # Check whether the input is an IP or hostmask or a nickname + search_str = trigger.group(3) # Groups 3-6 = command args 1-4 decide = ['.', ':'] - if any(x in trigger.group(2) for x in decide): + if any(x in search_str for x in decide): # It's an IP/hostname! - query = trigger.group(2).strip() + query = search_str.strip() else: - # Need to get the host for the username - username = trigger.group(2).strip() - user_in_botdb = bot.users.get(username) - if user_in_botdb is not None: - query = user_in_botdb.host - - # Sanity check - sometimes user information isn't populated yet - if query is None: - return bot.say("I don't know that user's host.") + # Need to get the ip for the username + nick = search_str.strip().lower() + if user_in_botdb := bot.users.get(nick): + if hasattr(user_in_botdb, "ip") and user_in_botdb.ip: + query = user_in_botdb.ip + # Sanity check - sometimes user information isn't populated yet + else: + return bot.say("I don't know that user's IP.") else: + #TODO TODO TODO get from DB return bot.say("I\'m not aware of this user.") - db_path = _find_geoip_db(bot) - if db_path is False: - LOGGER.error('Can\'t find (or download) usable GeoIP database.') - bot.say('Sorry, I don\'t have a GeoIP database to use for this lookup.') - return False + ex = get_exemption(query).lower() - if ':' in query: - try: - socket.inet_pton(socket.AF_INET6, query) - except (OSError, socket.error): # Python 2/3 compatibility - return bot.say("[IP/Host Lookup] Unable to resolve IP/Hostname") - elif '.' in query: - try: - socket.inet_pton(socket.AF_INET, query) - except (socket.error, socket.herror): + if ex: + irccloud = "irccloud" in ex + mibbit = "mibbit" in ex + if not any((irccloud, mibbit)): + db_path = _find_geoip_db(bot) + if db_path is False: + LOGGER.error('Can\'t find (or download) usable GeoIP database.') + bot.say('Sorry, I don\'t have a GeoIP database to use for this lookup.') + return False + + if ':' in query: try: - query = socket.getaddrinfo(query, None)[0][4][0] - except socket.gaierror: + socket.inet_pton(socket.AF_INET6, query) + except (OSError, socket.error): # Python 2/3 compatibility return bot.say("[IP/Host Lookup] Unable to resolve IP/Hostname") + elif '.' in query: + try: + socket.inet_pton(socket.AF_INET, query) + except (socket.error, socket.herror): + try: + query = socket.getaddrinfo(query, None)[0][4][0] + except socket.gaierror: + return bot.say("[IP/Host Lookup] Unable to resolve IP/Hostname") + else: + return bot.say("[IP/Host Lookup] Unable to resolve IP/Hostname") + + city = geoip2.database.Reader(os.path.join(db_path, 'GeoLite2-City.mmdb')) + asn = geoip2.database.Reader(os.path.join(db_path, 'GeoLite2-ASN.mmdb')) + host = socket.getfqdn(query) + try: + city_response = city.city(query) + asn_response = asn.asn(query) + except geoip2.errors.AddressNotFoundError: + return bot.say("[IP/Host Lookup] The address is not in the database.") + + response = "[IP/Host Lookup]" + + if irccloud: + response += " IP belongs to IRCCloud; no location data available" + return bot.say(response) + elif mibbit: + response += " IP belongs to mibbit; no location data available" + return bot.say(response) else: - return bot.say("[IP/Host Lookup] Unable to resolve IP/Hostname") + response += f" | IP meets exemption [{ex}] |" + # Still look up an IP that's exempt for other reasons - city = geoip2.database.Reader(os.path.join(db_path, 'GeoLite2-City.mmdb')) - asn = geoip2.database.Reader(os.path.join(db_path, 'GeoLite2-ASN.mmdb')) - host = socket.getfqdn(query) - try: - city_response = city.city(query) - asn_response = asn.asn(query) - except geoip2.errors.AddressNotFoundError: - return bot.say("[IP/Host Lookup] The address is not in the database.") + if full: + response += " Hostname: %s |" % host - response = "[IP/Host Lookup] Hostname: %s" % host try: - response += " | Location: %s" % city_response.country.name + response_loc = " Location: %s" % city_response.country.name + region = city_response.subdivisions.most_specific.name + response_loc += " | Region: %s" % region if region else "" + + if full: + city = city_response.city.name + response_loc += " | City: %s" % city if city else "" + + response += response_loc + except AttributeError: - response += ' | Location: Unknown' - - region = city_response.subdivisions.most_specific.name - response += " | Region: %s" % region if region else "" - city = city_response.city.name - response += " | City: %s" % city if city else "" - isp = "AS" + str(asn_response.autonomous_system_number) + \ - " " + asn_response.autonomous_system_organization - response += " | ISP: %s" % isp if isp else "" + response += ' Location: Unknown' + + try: + isp = "AS" + str(asn_response.autonomous_system_number) + \ + " " + asn_response.autonomous_system_organization + response += " | ISP: %s" % isp if isp else "" + except: + response += ' ISP: Unknown' + + force_lookup = trigger.group(4) == "lookup" + res = None + try: + res = retrieve_score(bot, query, nick, force_lookup) + except Exception as e: + LOGGER.error(f"Couldn't look up IP {query} because {e}") + if res: + response += f" | Score: {res[0]} Proxy: {res[1]} Recent abuse detected: {res[2]}" + elif not force_lookup: + # Use search_str to avoid leaking an IP + response += f" | To retrieve IP score run '.ip {search_str} lookup'" bot.say(response) From ec720873d88545c77000a0787ed80c62dc614946 Mon Sep 17 00:00:00 2001 From: Kufat Date: Sat, 30 Jan 2021 19:19:11 -0500 Subject: [PATCH 02/11] Tidy comment re URL params --- sopel/modules/ip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sopel/modules/ip.py b/sopel/modules/ip.py index 1569f348f5..f3505a90c8 100644 --- a/sopel/modules/ip.py +++ b/sopel/modules/ip.py @@ -173,9 +173,9 @@ def fetch_IPQS_score( ) -> tuple[float, bool, bool]: #score, is proxy, has recent abuse flag set '''Perform lookup on a specific IP adress using ipqualityscore.com''' params = urllib.parse.urlencode({ - # Case to lower + handle None and other garbage 'allow_public_access_points': "true" if allow_public_access_points else "false", 'strictness': int(strictness), + # lowercase + handle None and other non-bool garbage 'fast': "true" if fast else "false", 'mobile': "true" if mobile else "false", }) From 6a58611750498f604338d6a417967823f4f23a1f Mon Sep 17 00:00:00 2001 From: Kufat Date: Sat, 30 Jan 2021 21:06:12 -0500 Subject: [PATCH 03/11] Apply suggestions from code review Co-authored-by: Ross Williams --- sopel/modules/emailcheck.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sopel/modules/emailcheck.py b/sopel/modules/emailcheck.py index 6bb5e9d80c..beceb7d233 100644 --- a/sopel/modules/emailcheck.py +++ b/sopel/modules/emailcheck.py @@ -28,7 +28,7 @@ def get_exemption(ip): return "Can't access exemptions; failing safe" EMAIL_REGEX = re.compile(r"([a-zA-Z0-9_.+-]+)@([a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)") -IRCCLOUD_USER_REGEX = re.compile(r"[us]id[\d]{4,}") +IRCCLOUD_USER_REGEX = re.compile(r"[us]id[0-9]{4,}") DOMAIN_LEN = 50 DEFAULT_EXEMPT_SUFFIXES = { @@ -237,7 +237,7 @@ def retrieve_score(bot, email, nick): store_email_score_in_db(session, email, nick, IPQSresult) return IPQSresult else: #Shouldn't be possible - raise RuntimeError("Couldn't retrieve IPQS!") + raise RuntimeError(f"Couldn't retrieve IPQS for {email}!") except SQLAlchemyError: session.rollback() raise @@ -279,4 +279,3 @@ def handle_ns_register(bot, trigger): except: alert(f"Lookup for f{nick} with email @f{domain} failed! Keep an eye on them.") - From bbaa12a99a08d82e361ac8e0e67d380250c01e06 Mon Sep 17 00:00:00 2001 From: Kufat Date: Sat, 30 Jan 2021 21:14:21 -0500 Subject: [PATCH 04/11] Address add'l code review issues --- sopel/modules/emailcheck.py | 16 ++++++++-------- sopel/modules/ip.py | 9 +++++---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/sopel/modules/emailcheck.py b/sopel/modules/emailcheck.py index beceb7d233..bf71883ac6 100644 --- a/sopel/modules/emailcheck.py +++ b/sopel/modules/emailcheck.py @@ -173,10 +173,10 @@ def protect_chans(bot): def malicious_response(bot, nick: str, email): fdrop(bot, nick) add_badmail(bot, email) - say(f"You have been temporarily banned from this network because {email.domain} " - "has a history of spam or abuse, and/or is a disposable email domain. " - "If this is a legitimate domain, contact staff for assistance.", - nick.lower()) + bot.say(f"You have been temporarily banned from this network because {email.domain} " + "has a history of spam or abuse, and/or is a disposable email domain. " + "If this is a legitimate domain, contact staff for assistance.", + nick.lower()) gline_or_kill(bot, nick, config.emailcheck.gline_time) protect_chans(bot) alert(bot, f"ALERT: User {nick} attempted to register a nick with disposable/spam domain {email.domain}!") @@ -184,9 +184,9 @@ def malicious_response(bot, nick: str, email): def disallow_response(bot, nick: str, email): fdrop(bot, nick) add_badmail(bot, email) - say(f"Your registration has been disallowed because {email.domain} appears to be suspicious. " - "If this is a legitimate domain, contact staff for assistance.", - nick.lower()) + bot.say(f"Your registration has been disallowed because {email.domain} appears to be suspicious. " + "If this is a legitimate domain, contact staff for assistance.", + nick.lower()) alert(bot, f"WARNING: User {nick} attempted to register a nick with suspicious domain {email.domain}.") def fetch_IPQS_email_score( @@ -255,7 +255,7 @@ def check_email(bot, email, nick): # ExampleAccount REGISTER: ExampleNick to foo@example.com # (note the 0x02 bold chars) -@module.rule(r'(\S*)\s*REGISTER: ?([\S]+?)? to ?(\S+)@(\S+?)?$') +@module.rule(r'(\S*)\s*REGISTER: \u0002?([\S]+?)\u0002? to \u0002?(\S+)@(\S+?)\u0002?$') @module.event("PRIVMSG") @module.priority("high") def handle_ns_register(bot, trigger): diff --git a/sopel/modules/ip.py b/sopel/modules/ip.py index f3505a90c8..e289db2b92 100644 --- a/sopel/modules/ip.py +++ b/sopel/modules/ip.py @@ -389,15 +389,16 @@ def handle_snotice_ren(bot, trigger): @module.require_privilege(module.OP) @module.commands('ip_exempt') -@module.example('.ip_exempt 8.8.8.8') +@module.example('.ip_exempt 8.8.8.8 Known user example123\'s bouncer') def ip_exempt(bot, trigger): - if not trigger.group(3): + if not ipstr := trigger.group(3): # arg 1 return bot.reply("You must specify an IP or range in CIDR format to exempt.") - elif not trigger.group(4): + elif not trigger.group(4): # arg 2 must exist; need at least one word of desc return bot.reply("You must specify a reason for the exemption.") - ipstr = trigger.group(3) if '*' in ipstr: return bot.reply("Use CIDR format (1.2.3.0/24) rather than wildcard format (1.2.3.*)") + # Desc may be multiple words. Group 2 is all arguments. Strip off the first one and + # keep the rest. Based on code from the tell.py module. reason = trigger.group(2).lstrip(ipstr).lstrip() try: _add_exemption(ipstr, reason) From 66a5906292b56f144b339c99a688d0b841ac3acb Mon Sep 17 00:00:00 2001 From: Kufat Date: Sat, 30 Jan 2021 23:51:12 -0500 Subject: [PATCH 05/11] Various bugfixes --- sopel/modules/emailcheck.py | 85 ++++++++++++++++------- sopel/modules/ip.py | 133 ++++++++++++++++++++++-------------- 2 files changed, 142 insertions(+), 76 deletions(-) diff --git a/sopel/modules/emailcheck.py b/sopel/modules/emailcheck.py index bf71883ac6..4bec0ad5ab 100644 --- a/sopel/modules/emailcheck.py +++ b/sopel/modules/emailcheck.py @@ -19,14 +19,21 @@ from sopel.tools import events, target, Identifier from sqlalchemy import Column, String, Float, Boolean, TIMESTAMP +from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.declarative import declarative_base try: from ip import get_exemption + from ip import ipqs_lock except: + import threading + def get_exemption(ip): return "Can't access exemptions; failing safe" + ipqs_lock = threading.Lock() + + EMAIL_REGEX = re.compile(r"([a-zA-Z0-9_.+-]+)@([a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)") IRCCLOUD_USER_REGEX = re.compile(r"[us]id[0-9]{4,}") DOMAIN_LEN = 50 @@ -44,6 +51,9 @@ def get_exemption(ip): BASE = declarative_base() +safe_mode = True +db_populated = False + #SQLAlchemy container class class KnownEmails(BASE): __tablename__ = 'known_emails' @@ -56,10 +66,9 @@ class KnownEmails(BASE): class EmailCheckSection(StaticSection): IPQS_key = ValidatedAttribute('IPQS_key') - disallow_threshold = ValidatedAttribute("disallow_threshold", parse=float) - malicious_threshold = ValidatedAttribute("malicious_threshold", parse=float) - gline_time = ValidatedAttribute('gline_time') - #TODO; just hard-coded ones for now + disallow_threshold = ValidatedAttribute("disallow_threshold", parse=float, default=50.0) + malicious_threshold = ValidatedAttribute("malicious_threshold", parse=float, default=75.0) + gline_time = ValidatedAttribute('gline_time', default="24h") exempt_suffixes = ListAttribute("exempt_suffixes") warn_chans = ListAttribute("warn_chans") protect_chans = ListAttribute("protect_chans") @@ -69,15 +78,12 @@ def configure(config): config.emailcheck.configure_setting('IPQS_key', 'Access key for IPQS service') config.emailcheck.configure_setting('disallow_threshold', - 'Addresses with scores >= this will be disallowed; no punishment', - default=50.0) + 'Addresses with scores >= this will be disallowed; no punishment') config.emailcheck.configure_setting('malicious_threshold', - 'Addresses with scores >= this will be interpreted as attacks', - default=75.0) + 'Addresses with scores >= this will be interpreted as attacks') config.emailcheck.configure_setting('gline_time', 'Users attempting to register with malicious addresses will be ' - 'glined for this priod of time.', - default="24h") + 'glined for this priod of time.') config.emailcheck.configure_setting('exempt_suffixes', 'Suffixes (TLD, whole domain, etc.) to exempt from checking') config.emailcheck.configure_setting('warn_chans', @@ -108,26 +114,38 @@ class DomainInfo: flag_recent_abuse: bool def alert(bot, alert_msg: str, log_err: bool = False): - for channel in config.emailcheck.warn_chans: + for channel in bot.config.emailcheck.warn_chans: bot.say(alert_msg, channel) if log_err: LOGGER.error(alert_msg) def add_badmail(bot, email): #Right now we're BADMAILing whole domains. This might change. - bot.write("NICKSERV", "badmail", "add", f'*@{email.domain}') + if safe_mode: + LOGGER.info(f"SAFE MODE: Would badmail {email}") + else: + bot.write("NICKSERV", "badmail", "add", f'*@{email.domain}') def fdrop(bot, nick: str): - bot.write("NICKSERV", "fdrop", nick.lower()) + if safe_mode: + LOGGER.info(f"SAFE MODE: Would fdrop {nick}") + else: + bot.write("NICKSERV", "fdrop", nick.lower()) def gline_ip(bot, ip: str, duration: str): - bot.write("GLINE", f'*@{ip}', duration, KILL_STR) + if safe_mode: + LOGGER.info(f"SAFE MODE: Would gline {ip} for {duration}") + else: + bot.write("GLINE", f'*@{ip}', duration, KILL_STR) def gline_username(bot, nick: str, duration: str): if known_user := bot.users.get(Identifier(nick)): username = known_user.user.lower() # Should already be lowercase if IRCCLOUD_USER_REGEX.match(username): - bot.write("GLINE", f'{username}@*', duration, KILL_STR) + if safe_mode: + LOGGER.info(f"SAFE MODE: Would gline {username} for {duration}") + else: + bot.write("GLINE", f'{username}@*', duration, KILL_STR) return else: alert(bot, f"User {nick} had unexpected non-IRCCloud username {username}", true) @@ -136,7 +154,10 @@ def gline_username(bot, nick: str, duration: str): kill_nick(bot, nick) # Something went wrong with G-line, so fall back to /kill def kill_nick(bot, nick: str): - bot.write("KILL", nick.lower(), KILL_STR) + if safe_mode: + LOGGER.info(f"SAFE MODE: Would kill {nick}") + else: + bot.write("KILL", nick.lower(), KILL_STR) def gline_strategy(bot, nick): if (known_user := bot.users.get(Identifier(nick))): @@ -167,7 +188,10 @@ def gline_or_kill(bot, nick: str, duration: str): kill_nick(bot, nick) # duration ignored def protect_chans(bot): - for chan in config.emailcheck.protect_chans: + if safe_mode: + LOGGER.info(f"SAFE MODE: Would protect chans") + return + for chan in bot.config.emailcheck.protect_chans: bot.write("MODE", chan, "+R") def malicious_response(bot, nick: str, email): @@ -177,7 +201,7 @@ def malicious_response(bot, nick: str, email): "has a history of spam or abuse, and/or is a disposable email domain. " "If this is a legitimate domain, contact staff for assistance.", nick.lower()) - gline_or_kill(bot, nick, config.emailcheck.gline_time) + gline_or_kill(bot, nick, bot.config.emailcheck.gline_time) protect_chans(bot) alert(bot, f"ALERT: User {nick} attempted to register a nick with disposable/spam domain {email.domain}!") @@ -228,12 +252,16 @@ def store_email_score_in_db(session, email, nick, IPQSresult): session.commit() def retrieve_score(bot, email, nick): - session = bot.db.ssession() + session = bot.db.session() try: + global db_populated + if not db_populated: + BASE.metadata.create_all(bot.db.engine) + db_populated = True if retval := get_email_score_from_db(session, email): return retval else: - if IPQSresult := fetch_IPQS_email_score(email, config.emailcheck.IPQS_key): + if IPQSresult := fetch_IPQS_email_score(email, bot.config.emailcheck.IPQS_key): store_email_score_in_db(session, email, nick, IPQSresult) return IPQSresult else: #Shouldn't be possible @@ -241,8 +269,6 @@ def retrieve_score(bot, email, nick): except SQLAlchemyError: session.rollback() raise - finally: - session.remove() def check_email(bot, email, nick): if any(map(email.endswith, DEFAULT_EXEMPT_SUFFIXES)): @@ -253,14 +279,21 @@ def check_email(bot, email, nick): else: return retrieve_score(bot, email, nick) +@module.require_owner +@module.commands('toggle_safe_email') +def toggle_safe(bot, trigger): + global safe_mode + safe_mode = not safe_mode + return bot.reply(f"Email check module safe mode now {'on' if safe_mode else 'off'}") + # ExampleAccount REGISTER: ExampleNick to foo@example.com # (note the 0x02 bold chars) @module.rule(r'(\S*)\s*REGISTER: \u0002?([\S]+?)\u0002? to \u0002?(\S+)@(\S+?)\u0002?$') @module.event("PRIVMSG") @module.priority("high") def handle_ns_register(bot, trigger): - if "nickserv" != trigger.sender.lower(): - LOGGER.warning(f"Fake registration notice from {trigger.sender.lower()}!") + if "nickserv" != trigger.nick.lower(): + LOGGER.warning(f"Fake registration notice from {trigger.nick.lower()}!") return #It's really from nickserv. _, nick, email_user, email_domain = trigger.groups() @@ -268,10 +301,10 @@ def handle_ns_register(bot, trigger): try: if res := check_email(bot, email_user, email_domain, nick): #may be None, in which case we're done if res.flag_disposable or ( - res.score >= config.emailcheck.malicious_threshold): + res.score >= bot.config.emailcheck.malicious_threshold): malicious_response(bot, nick, email) elif res.flag_recent_abuse or ( - res.score >= config.emailcheck.disallow_threshold): + res.score >= bot.config.emailcheck.disallow_threshold): disallow_response(bot, nick, email) else: #already logged server response diff --git a/sopel/modules/ip.py b/sopel/modules/ip.py index e289db2b92..3d1c553625 100644 --- a/sopel/modules/ip.py +++ b/sopel/modules/ip.py @@ -19,6 +19,7 @@ import geoip2.database import re +import threading import urllib.request, json import sqlalchemy.sql @@ -33,6 +34,7 @@ from sopel.tools import web, events, target, Identifier from sqlalchemy import Column, Integer, String, Float, Boolean, TIMESTAMP, Text +from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.declarative import declarative_base IRCCLOUD_IP = [ @@ -85,6 +87,10 @@ BASE = declarative_base() +safe_mode = True +db_populated = False +ipqs_lock = threading.Lock() + # This table will only receive inserts, not updates. class KnownIPs(BASE): __tablename__ = 'known_ips' @@ -116,8 +122,8 @@ class GeoipSection(StaticSection): GeoIP_db_path = FilenameAttribute('GeoIP_db_path', directory=True) """Path of the directory containing the GeoIP database files.""" IPQS_key = ValidatedAttribute('IPQS_key') - warn_threshold = ValidatedAttribute('warn_threshold', parse=float) - malicious_threshold = ValidatedAttribute('malicious_threshold', parse=float) + warn_threshold = ValidatedAttribute('warn_threshold', parse=float, default=50.0) + malicious_threshold = ValidatedAttribute('malicious_threshold', parse=float, default=70.0) warn_chans = ListAttribute("warn_chans") protect_chans = ListAttribute("protect_chans") @@ -128,11 +134,9 @@ def configure(config): config.ip.configure_setting('IPQS_key', 'Access key for IPQS service') config.ip.configure_setting('warn_threshold', - 'Addresses with scores >= this will generate an alert', - default=50.0) + 'Addresses with scores >= this will generate an alert') config.ip.configure_setting('malicious_threshold', - 'Addresses with scores >= this will be z-lined', - default=70.0) + 'Addresses with scores >= this will be z-lined') config.ip.configure_setting('warn_chans', 'List of channels to warn when a suspicious user is detected. ' 'May be empty.') @@ -140,34 +144,40 @@ def configure(config): 'List of channels to +R after malicious attempt to reg. ' 'May be empty.') +safe_mode = True def setup(bot): bot.config.define_section('ip', GeoipSection) def alert(bot, alert_msg: str, log_err = False): - for channel in config.ip.warn_chans: + for channel in bot.config.ip.warn_chans: bot.say(alert_msg, channel) if log_err: LOGGER.error(alert_msg) def get_exemption(host): - if isinstance(host, ip.IPv4Address) or isinstance(host, ip.IPv6Address): - host = ip + ip = None + if isinstance(host, ipaddress.IPv4Address) or isinstance(host, ipaddress.IPv6Address): + ip = host else: try: ip = ipaddress.ip_address(socket.getaddrinfo(host, None)[0][4][0]) except: raise + if not ip.is_global: + LOGGER.warn(f"Non-global IP {ip} seen.") + return "Non-global IP; internal network, localhost, etc." for network, reason in exemptions.items(): if ip in network: return reason return None -def fetch_IPQS_score( +def fetch_IPQS_ip_score( ip_addr: typing.Union[ipaddress.IPv4Address, ipaddress.IPv6Address], key: str, allow_public_access_points: bool = True, - strictness: int = 1, + lighter_penalties: bool = true, + strictness: int = 2, fast: bool = False, mobile: bool = False ) -> tuple[float, bool, bool]: #score, is proxy, has recent abuse flag set @@ -178,17 +188,18 @@ def fetch_IPQS_score( # lowercase + handle None and other non-bool garbage 'fast': "true" if fast else "false", 'mobile': "true" if mobile else "false", + 'lighter_penalties': "true" if lighter_penalties else "false", }) # ip_addr sourced from server, not user, so sanitization already done with urllib.request.urlopen( f"https://ipqualityscore.com/api/json/ip/{key}/{str(ip_addr)}?{params}") as url: data = json.loads(url.read().decode()) - LOGGER.info(data) + LOGGER.debug(data) if not data['success']: errstr = f"{ip_addr} lookup failed with {data['message']}" LOGGER.error(errstr) raise RuntimeError(errstr) - return (data['score'], data["proxy"], data["recent_abuse"]) + return (data['fraud_score'], data["proxy"], data["recent_abuse"]) def get_ip_score_from_db(session, ip): query_result = session.query(KnownIPs)\ @@ -198,36 +209,40 @@ def get_ip_score_from_db(session, ip): #Any known problematic provider should've been BADMAILed by now, but... return (query_result.score, query_result.flag_recent_abuse, - query_result.flag_proxy + query_result.flag_is_proxy ) def store_ip_score_in_db(session, ip, nick, IPQSresult): - new_known_ip = KnownEmails(ip= ip, - score= IPQSresult[0], - flag_recent_abuse= IPQSresult[1], - flag_is_proxy= IPQSresult[2]) + new_known_ip = KnownIPs(ip= ip, + score= IPQSresult[0], + flag_recent_abuse= IPQSresult[1], + flag_is_proxy= IPQSresult[2]) session.add(new_known_ip) session.commit() def retrieve_score(bot, ip, nick, do_fetch = True): - session = bot.db.ssession() - try: - if retval := get_ip_score_from_db(session, ip): - return retval - elif do_fetch: - if IPQSresult := fetch_IPQS_ip_score(ip, config.emailcheck.IPQS_key): - store_ip_score_in_db(session, ip, nick, IPQSresult) - return IPQSresult - else: #Shouldn't be possible - raise RuntimeError("Couldn't retrieve IPQS!") - else: - # If do_fetch is false, this is a best-effort request and shouldn't use up a query - return None - except SQLAlchemyError: - session.rollback() - raise - finally: - session.remove() + with ipqs_lock: + LOGGER.debug(f"Lock acquired. Beginning lookup for {str(ip)}") + global db_populated + session = bot.db.session() + try: + if not db_populated: + BASE.metadata.create_all(bot.db.engine) + db_populated = True + if retval := get_ip_score_from_db(session, ip): + return retval + elif do_fetch: + if IPQSresult := fetch_IPQS_ip_score(ip, bot.config.emailcheck.IPQS_key): + store_ip_score_in_db(session, ip, nick, IPQSresult) + return IPQSresult + else: #Shouldn't be possible + raise RuntimeError("Couldn't retrieve IPQS!") + else: + # If do_fetch is false, this is a best-effort request and shouldn't use up a query + return None + except SQLAlchemyError: + session.rollback() + raise def _add_exemption(ip, reason): exemptions[ipaddress.ip_network(ip)] = reason @@ -289,24 +304,32 @@ def _find_geoip_db(bot): def populate_user(bot, user, ip, host, nick): LOGGER.debug('Adding: %s!%s@%s with IP %s', nick, user, host, ip) - - user = bot.users.get(nick) or target.User(nick, user, host) + user = bot.users.get(nick) or target.User(Identifier(nick), user, host) if ip: user.ip = ip # Add nonstandard field bot.users[nick] = user # no-op if user was in users, needed otherwise +def zline(bot, ip, nick, duration): + if safe_mode: + LOGGER.info(f"SAFE MODE: Would zline {ip} for {duration}") + else: + bot.write("ZLINE", ip, duration, f":Auto z-line {nick}.") + def examine_user(bot, user, ip, host, nick): populate_user(bot, user, ip, host, nick) res = retrieve_score(bot, ip, nick) if res: score, is_proxy, is_recent_abuse = res - if( is_prox or is_recent_abuse or score >= config.ip.malicious_threshold ): - alert(bot, f"Ops: Nick {nick} has abuse score {score}, proxy: {is_prox}, " - "recent_abuse: {is_recent_abuse}; z-lining!") - bot.write("ZLINE", ip, "24h", f":Auto z-line {nick}.") + if( is_proxy or is_recent_abuse or score >= bot.config.ip.malicious_threshold ): + alert(bot, f"{"Orps" if safe_mode else "Ops"}" + f": Nick {nick} has abuse score {score}, proxy: {is_proxy}, " + f"recent_abuse: {is_recent_abuse}; z-lining!") + duration = "24h" + zline(bot, ip, nick, duration) protect_chans(bot) - elif score >= config.ip.warn_threshold: - alert(bot, f"Ops: Nick {nick} has abuse score {score}; keep an eye on them.") + elif score >= bot.config.ip.warn_threshold: + alert(bot, f"{"Orps" if safe_mode else "Ops"}" + f": Nick {nick} has abuse score {score}; keep an eye on them.") return res @module.event(events.RPL_WHOSPCRPL) @@ -322,6 +345,9 @@ def recv_whox_ip(bot, trigger): if len(trigger.args) != 6: return LOGGER.warning('While populating the IP DB, a WHO response was malformed.') _, _, user, ip, host, nick = trigger.args + if get_exemption(ip): + LOGGER.debug(f"Exempt IP {ip} for {nick}") + return examine_user(bot, user, ip, host, nick) @module.event(events.RPL_ENDOFWHO) @@ -358,14 +384,13 @@ def handle_snotice_conn(bot, trigger): #Only servers may have '.' in the sender name, so this isn't spoofable if "scpwiki.com" in trigger.sender: nick, user, host, ip = trigger.groups() - examine_user(bot, user, ip, None, nick) #cloaked host not known - #Be **certain** we don't waste our lookups on irccloud - if any(host.endswith(s) for s in (".irccloud.com", ".mibbit.com")): - return # We need to check if the IP is in any exempt CIDR ranges if get_exemption(ip): return - + #Be **certain** we don't waste our lookups on irccloud + if any(host.endswith(s) for s in (".irccloud.com", ".mibbit.com")): + LOGGER.error(f"{host} slipped through get_exemption()") + return res = examine_user(bot, user, ip, host, nick) if res: # Acted on above; just log here @@ -387,11 +412,19 @@ def handle_snotice_ren(bot, trigger): if olduser := bot.users.get(oldnick): populate_user(bot, olduser.user, olduser.ip, olduser.host, newnick) +@module.require_owner +@module.commands('toggle_safe_ip') +def toggle_safe(bot, trigger): + global safe_mode + safe_mode = not safe_mode + return bot.reply(f"IP module safe mode now {'on' if safe_mode else 'off'}") + @module.require_privilege(module.OP) @module.commands('ip_exempt') @module.example('.ip_exempt 8.8.8.8 Known user example123\'s bouncer') def ip_exempt(bot, trigger): - if not ipstr := trigger.group(3): # arg 1 + ipstr = trigger.group(3) # arg 1 + if not ipstr: return bot.reply("You must specify an IP or range in CIDR format to exempt.") elif not trigger.group(4): # arg 2 must exist; need at least one word of desc return bot.reply("You must specify a reason for the exemption.") From d159ee5ac5203eef48616bab77d18e66664119cc Mon Sep 17 00:00:00 2001 From: Kufat Date: Sun, 31 Jan 2021 01:07:42 -0500 Subject: [PATCH 06/11] Apply suggestions from code review Co-authored-by: Ammon Smith --- sopel/modules/emailcheck.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sopel/modules/emailcheck.py b/sopel/modules/emailcheck.py index 4bec0ad5ab..d35e678f9c 100644 --- a/sopel/modules/emailcheck.py +++ b/sopel/modules/emailcheck.py @@ -284,7 +284,7 @@ def check_email(bot, email, nick): def toggle_safe(bot, trigger): global safe_mode safe_mode = not safe_mode - return bot.reply(f"Email check module safe mode now {'on' if safe_mode else 'off'}") + return bot.reply(f"Email check module safe mode now {'ON' if safe_mode else 'OFF'}") # ExampleAccount REGISTER: ExampleNick to foo@example.com # (note the 0x02 bold chars) @@ -311,4 +311,3 @@ def handle_ns_register(bot, trigger): return LOGGER.debug(f'Registration of {nick} to {email} OK.') except: alert(f"Lookup for f{nick} with email @f{domain} failed! Keep an eye on them.") - From 24f6595d72e8d63cde6a35e0dc47859d5985e0d0 Mon Sep 17 00:00:00 2001 From: Kufat Date: Sun, 31 Jan 2021 20:31:28 -0500 Subject: [PATCH 07/11] Get rid of IPQS in favor of MaxMind and validator.pizza. Address most remaining review comments. --- sopel/modules/emailcheck.py | 208 ++++++++++++++------------------ sopel/modules/ip.py | 233 +++++++++++++++++++----------------- 2 files changed, 214 insertions(+), 227 deletions(-) diff --git a/sopel/modules/emailcheck.py b/sopel/modules/emailcheck.py index d35e678f9c..16b5305139 100644 --- a/sopel/modules/emailcheck.py +++ b/sopel/modules/emailcheck.py @@ -6,13 +6,18 @@ Licensed under the Eiffel Forum License 2. """ +import json import logging import re +import threading import urllib import sqlalchemy.sql +from collections import namedtuple from dataclasses import dataclass +from http import HTTPStatus +from typing import Tuple from sopel import db, module from sopel.config.types import FilenameAttribute, StaticSection, ValidatedAttribute, ListAttribute @@ -22,70 +27,45 @@ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.declarative import declarative_base -try: - from ip import get_exemption - from ip import ipqs_lock -except: - import threading +from .ip import get_exemption, sopel_session_scope - def get_exemption(ip): - return "Can't access exemptions; failing safe" - - ipqs_lock = threading.Lock() - - -EMAIL_REGEX = re.compile(r"([a-zA-Z0-9_.+-]+)@([a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)") IRCCLOUD_USER_REGEX = re.compile(r"[us]id[0-9]{4,}") DOMAIN_LEN = 50 -DEFAULT_EXEMPT_SUFFIXES = { - "@gmail.com", - "@hotmail.com", - "@protonmail.com", - ".edu" -} - KILL_STR = ":Use of disposable email service for nick registration" LOGGER = logging.getLogger(__name__) BASE = declarative_base() -safe_mode = True -db_populated = False +email_safe_mode = True + +pizza_lock = threading.Lock() + +ValidatorPizzaResponse = namedtuple('ValidatorPizzaResponse', + ['flag_valid', 'flag_disposable']) + +GLineStrategy = namedtuple('GLineStrategy', ['strategy', 'targer']) #SQLAlchemy container class class KnownEmails(BASE): __tablename__ = 'known_emails' - domain = Column(String(DOMAIN_LEN), primary_key=True) + domain = Column(String(DOMAIN_LEN), primary_key=True, index=True) first_nick = Column(String(40)) - score = Column(Float) - flag_disposable = Column(Boolean) - flag_recent_abuse = Column(Boolean) + flag_valid = Column(Boolean) + flag_disposable = Column(Boolean, nullable=False) first_seen = Column(TIMESTAMP, server_default=sqlalchemy.sql.func.now()) class EmailCheckSection(StaticSection): - IPQS_key = ValidatedAttribute('IPQS_key') - disallow_threshold = ValidatedAttribute("disallow_threshold", parse=float, default=50.0) - malicious_threshold = ValidatedAttribute("malicious_threshold", parse=float, default=75.0) - gline_time = ValidatedAttribute('gline_time', default="24h") - exempt_suffixes = ListAttribute("exempt_suffixes") - warn_chans = ListAttribute("warn_chans") - protect_chans = ListAttribute("protect_chans") + gline_time = ValidatedAttribute('gline_time', default='24h') + warn_chans = ListAttribute('warn_chans') + protect_chans = ListAttribute('protect_chans') def configure(config): config.define_section('emailcheck', EmailCheckSection) - config.emailcheck.configure_setting('IPQS_key', - 'Access key for IPQS service') - config.emailcheck.configure_setting('disallow_threshold', - 'Addresses with scores >= this will be disallowed; no punishment') - config.emailcheck.configure_setting('malicious_threshold', - 'Addresses with scores >= this will be interpreted as attacks') config.emailcheck.configure_setting('gline_time', 'Users attempting to register with malicious addresses will be ' 'glined for this priod of time.') - config.emailcheck.configure_setting('exempt_suffixes', - 'Suffixes (TLD, whole domain, etc.) to exempt from checking') config.emailcheck.configure_setting('warn_chans', 'List of channels to warn when a suspicious user is detected. ' 'May be empty.') @@ -95,6 +75,7 @@ def configure(config): def setup(bot): bot.config.define_section('emailcheck', EmailCheckSection) + BASE.metadata.create_all(bot.db.engine) @dataclass class Email: @@ -109,9 +90,8 @@ def __post_init__(self): @dataclass class DomainInfo: - score: float flag_disposable: bool - flag_recent_abuse: bool + flag_valid: bool def alert(bot, alert_msg: str, log_err: bool = False): for channel in bot.config.emailcheck.warn_chans: @@ -121,28 +101,28 @@ def alert(bot, alert_msg: str, log_err: bool = False): def add_badmail(bot, email): #Right now we're BADMAILing whole domains. This might change. - if safe_mode: + if email_safe_mode: LOGGER.info(f"SAFE MODE: Would badmail {email}") else: bot.write("NICKSERV", "badmail", "add", f'*@{email.domain}') def fdrop(bot, nick: str): - if safe_mode: + if email_safe_mode: LOGGER.info(f"SAFE MODE: Would fdrop {nick}") else: bot.write("NICKSERV", "fdrop", nick.lower()) def gline_ip(bot, ip: str, duration: str): - if safe_mode: + if email_safe_mode: LOGGER.info(f"SAFE MODE: Would gline {ip} for {duration}") else: bot.write("GLINE", f'*@{ip}', duration, KILL_STR) -def gline_username(bot, nick: str, duration: str): +def gline_irccloud(bot, nick: str, duration: str): if known_user := bot.users.get(Identifier(nick)): username = known_user.user.lower() # Should already be lowercase if IRCCLOUD_USER_REGEX.match(username): - if safe_mode: + if email_safe_mode: LOGGER.info(f"SAFE MODE: Would gline {username} for {duration}") else: bot.write("GLINE", f'{username}@*', duration, KILL_STR) @@ -154,7 +134,7 @@ def gline_username(bot, nick: str, duration: str): kill_nick(bot, nick) # Something went wrong with G-line, so fall back to /kill def kill_nick(bot, nick: str): - if safe_mode: + if email_safe_mode: LOGGER.info(f"SAFE MODE: Would kill {nick}") else: bot.write("KILL", nick.lower(), KILL_STR) @@ -167,20 +147,20 @@ def gline_strategy(bot, nick): if exemption: if "irccloud" in exemption.lower(): # IRCCloud special case: ban uid/sid - return ["gline_username", known_user.user] + return GLineStrategy("gline_irccloud", known_user.user) else: # Fully exempt, so no g-line return None else: # No exemption - return ["gline_ip", ip] + return GLineStrategy("gline_ip", ip) else: # Fail safely return None def gline_or_kill(bot, nick: str, duration: str): - if strategy := gline_strategy(bot, nick): - if strategy[0] == "gline_ip": - gline_ip(bot, strategy[1], duration) - elif strategy[0] == "gline_username": - gline_username(bot, strategy[1], duration) + if gline_strat := gline_strategy(bot, nick): + if gline_strat.strategy == "gline_ip": + gline_ip(bot, strategy.target, duration) + elif gline_strat.strategy == "gline_irccloud": + gline_irccloud(bot, strategy.target, duration) else: alert(bot, f"Unknown strategy {strategy} for nick {nick}", true) kill_nick(bot, nick) # safest option @@ -188,11 +168,12 @@ def gline_or_kill(bot, nick: str, duration: str): kill_nick(bot, nick) # duration ignored def protect_chans(bot): - if safe_mode: + if email_safe_mode: LOGGER.info(f"SAFE MODE: Would protect chans") return for chan in bot.config.emailcheck.protect_chans: bot.write("MODE", chan, "+R") + alert(bot, f"Setting {', '.join(bot.config.emailcheck.protect_chans)} +R") def malicious_response(bot, nick: str, email): fdrop(bot, nick) @@ -213,71 +194,67 @@ def disallow_response(bot, nick: str, email): nick.lower()) alert(bot, f"WARNING: User {nick} attempted to register a nick with suspicious domain {email.domain}.") -def fetch_IPQS_email_score( - email_addr: str, - key: str, - fast: bool = True - ) -> tuple[float, bool, bool]: #score, disposable, has recent abuse flag set - '''Perform lookup on a specific email adress using ipqualityscore.com''' - email_str = urllib.parse.quote(email_addr) - faststr = str(bool(fast)).lower() #lower + handle None and other garbage - params = urllib.parse.urlencode({'fast': faststr}) - with urllib.request.urlopen( - f"https://ipqualityscore.com/api/json/email/{key}/{email_str}?{params}") as url: - data = json.loads(url.read().decode()) - LOGGER.debug(data) - if not data['success']: - errstr = f"{email_addr} lookup failed with {data['message']}" +def fetch_validator_pizza_email_info(email_addr: str ) \ +-> Tuple[bool, bool]: #valid, disposable + '''Perform lookup on a specific email adress using validator.pizza''' + email_addr_str = urllib.parse.quote(str(email_addr)) + # Cloudflare likes headers. Sigh. + hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3', + 'Accept-Encoding': 'none', + 'Accept-Language': 'en-US,en;q=0.8', + 'Connection': 'keep-alive'} + urlstr = f"https://www.validator.pizza/email/{email_addr_str}" + req = urllib.request.Request(urlstr, headers=hdr) + try: + with pizza_lock, urllib.request.urlopen(req) as url: + data = json.loads(url.read().decode()) + LOGGER.debug(f"Received data from validator.pizza: {data}") + except urllib.error.HTTPError as err: + LOGGER.error(f"Error retrieving {urlstr}: {err.code}, {err.headers}") + raise + if data['status'] == HTTPStatus.OK: + return ValidatorPizzaResponse(data['mx'], data["disposable"]) + elif data['status'] == HTTPStatus.BAD_REQUEST: + # Address is invalid, assume typo + return (False, None) + elif data['status'] == HTTPStatus.TOO_MANY_REQUESTS: + # This is unlikely enough that I'm going to postpone dealing with it + raise RuntimeError("Hit request limit!") + else: # Anything other than 200/400/429 is out of spec + errstr = f"{email_addr} lookup failed with {data}" LOGGER.error(errstr) raise RuntimeError(errstr) - return (data['fraud_score'], data["disposable"], data["recent_abuse"]) -def get_email_score_from_db(session, email): +def get_email_info_from_db(session, email): query_result = session.query(KnownEmails)\ .filter(KnownEmails.domain == email.domain)\ .one_or_none() if query_result: #Any known problematic provider should've been BADMAILed by now, but... - return DomainInfo(query_result.score, - query_result.flag_disposable, - query_result.flag_recent_abuse) - -def store_email_score_in_db(session, email, nick, IPQSresult): - new_known_email = KnownEmails(domain= email.doman[:DOMAIN_LEN], - first_nick= nick, - score= IPQSresult[0], - flag_disposable= IPQSresult[1], - flag_recent_abuse= IPQSresult[2]) + return DomainInfo(query_result.flag_valid, + query_result.flag_disposable) + +def store_email_info_in_db(session, email, nick, result): + new_known_email = KnownEmails(domain= email.domain[:DOMAIN_LEN], + first_nick= nick, + flag_valid= result.flag_valid, + flag_disposable= result.flag_disposable) session.add(new_known_email) - session.commit() -def retrieve_score(bot, email, nick): - session = bot.db.session() - try: - global db_populated - if not db_populated: - BASE.metadata.create_all(bot.db.engine) - db_populated = True - if retval := get_email_score_from_db(session, email): +def retrieve_info_for_email(bot, email, nick): + session = bot.db.ssession() + with sopel_session_scope(bot) as session: + if retval := get_email_info_from_db(session, email): return retval else: - if IPQSresult := fetch_IPQS_email_score(email, bot.config.emailcheck.IPQS_key): - store_email_score_in_db(session, email, nick, IPQSresult) - return IPQSresult - else: #Shouldn't be possible - raise RuntimeError(f"Couldn't retrieve IPQS for {email}!") - except SQLAlchemyError: - session.rollback() - raise - -def check_email(bot, email, nick): - if any(map(email.endswith, DEFAULT_EXEMPT_SUFFIXES)): - #email is exempt - LOGGER.info(f'Email {email} used by {nick} is on the exemption list.') - return None # No lookup, no result - #Check database - else: - return retrieve_score(bot, email, nick) + if result := fetch_validator_pizza_email_info(email): + store_email_info_in_db(session, email, nick, result) + return result + else: + #Should either return or throw + raise RuntimeError(f"validator.pizza failed for email: {email}") @module.require_owner @module.commands('toggle_safe_email') @@ -299,15 +276,16 @@ def handle_ns_register(bot, trigger): _, nick, email_user, email_domain = trigger.groups() email = Email(email_user, email_domain) try: - if res := check_email(bot, email_user, email_domain, nick): #may be None, in which case we're done - if res.flag_disposable or ( - res.score >= bot.config.emailcheck.malicious_threshold): + # check_email() may return None, in which case we're done + if res := retrieve_info_for_email(bot, email, nick): + if res.flag_disposable: malicious_response(bot, nick, email) - elif res.flag_recent_abuse or ( - res.score >= bot.config.emailcheck.disallow_threshold): + elif not res.flag_valid : disallow_response(bot, nick, email) else: #already logged server response return LOGGER.debug(f'Registration of {nick} to {email} OK.') except: - alert(f"Lookup for f{nick} with email @f{domain} failed! Keep an eye on them.") + alert(bot, f"Lookup for {nick} with email @{email_domain} failed! " + "Keep an eye on them.") + raise diff --git a/sopel/modules/ip.py b/sopel/modules/ip.py index 3d1c553625..c8c6286875 100644 --- a/sopel/modules/ip.py +++ b/sopel/modules/ip.py @@ -24,10 +24,11 @@ import sqlalchemy.sql +from contextlib import contextmanager from random import randint from urllib.request import urlretrieve -#from minfraud import Client +from minfraud import Client from sopel import module from sopel.config.types import FilenameAttribute, StaticSection, ValidatedAttribute, ListAttribute @@ -37,6 +38,8 @@ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.declarative import declarative_base +IP_MAX_LEN = 45 # e.g. v4-mapped-v6 0000:0000:0000:0000:0000:ffff:192.168.100.228 + IRCCLOUD_IP = [ "2001:67c:2f08::/48", "2a03:5180:f::/64", @@ -74,6 +77,12 @@ ("2604:a880:2:d0::250:9001", "Approved bot (Idlebot, aismallard)"), ("91.132.86.177", "bluesoul's bouncer"), ("2001:470:1f07:13b::/64", "kufat's tunnelbroker"), + ("2604:a880:2:d0::/64", "noracodes"), + ("2a01:4f8:202:62c8::/64", "grumble"), + ("78.46.73.141", "hooloovoo"), + ("67.205.43.220", "carolynn ivy, dreamhost"), + ("212.47.230.56", "skee, token.ro"), + ("2600:3c01::/64", "atomicthumbs"), ] LOGGER = logging.getLogger(__name__) @@ -87,22 +96,21 @@ BASE = declarative_base() -safe_mode = True -db_populated = False -ipqs_lock = threading.Lock() +ip_safe_mode = True +MaxMind_lock = threading.Lock() +client = None # This table will only receive inserts, not updates. class KnownIPs(BASE): __tablename__ = 'known_ips' - ip = Column(String(50), primary_key=True, unique=True, index=True) - score = Column(Float) - flag_recent_abuse = Column(Boolean) - flag_is_proxy = Column(Boolean) + ip = Column(String(IP_MAX_LEN), primary_key=True, unique=True, index=True) + score = Column(Float, nullable=False) insert_time = Column(TIMESTAMP, server_default=sqlalchemy.sql.func.now()) +# TODO deferred feature, hardcoded list for now class ExemptIPs(BASE): __tablename__ = 'exempt_ips' - ip = Column(String(50), primary_key=True, unique=True, index=True) + ip = Column(String(IP_MAX_LEN), primary_key=True, unique=True, index=True) # Define type of exemption. # GLine username e.g. uid123@* if true, no g-line at all if false. For IRCCloud. gline_username = Column(Boolean) @@ -121,22 +129,28 @@ class KnownUsers(BASE): class GeoipSection(StaticSection): GeoIP_db_path = FilenameAttribute('GeoIP_db_path', directory=True) """Path of the directory containing the GeoIP database files.""" - IPQS_key = ValidatedAttribute('IPQS_key') + MaxMind_account_num = ValidatedAttribute('MaxMind_account_num', parse=int) + MaxMind_key = ValidatedAttribute('MaxMind_key') + sno_string = ValidatedAttribute('sno_string') warn_threshold = ValidatedAttribute('warn_threshold', parse=float, default=50.0) malicious_threshold = ValidatedAttribute('malicious_threshold', parse=float, default=70.0) - warn_chans = ListAttribute("warn_chans") - protect_chans = ListAttribute("protect_chans") + warn_chans = ListAttribute('warn_chans') + protect_chans = ListAttribute('protect_chans') def configure(config): config.define_section('ip', GeoipSection) config.ip.configure_setting('GeoIP_db_path', 'Path of the GeoIP db files') - config.ip.configure_setting('IPQS_key', - 'Access key for IPQS service') + config.ip.configure_setting('MaxMind_account_num', + 'Account number for MaxMind service') + config.ip.configure_setting('MaxMind_key', + 'Access key for MaxMind service') config.ip.configure_setting('warn_threshold', 'Addresses with scores >= this will generate an alert') config.ip.configure_setting('malicious_threshold', 'Addresses with scores >= this will be z-lined') + config.ip.configure_setting('sno_string', + 'String to look for in server notices to ensure legitimacy') config.ip.configure_setting('warn_chans', 'List of channels to warn when a suspicious user is detected. ' 'May be empty.') @@ -144,10 +158,26 @@ def configure(config): 'List of channels to +R after malicious attempt to reg. ' 'May be empty.') -safe_mode = True - def setup(bot): + global client bot.config.define_section('ip', GeoipSection) + BASE.metadata.create_all(bot.db.engine) + client = Client(bot.config.ip.MaxMind_account_num, + bot.config.ip.MaxMind_key) + +@contextmanager +def sopel_session_scope(bot): + """Provide a transactional scope around a series of operations.""" + session = bot.db.ssession() + try: + yield session + session.commit() + except SQLAlchemyError as e: + LOGGER.error(str(e)) + session.rollback() + raise + finally: + bot.db.ssession.remove() def alert(bot, alert_msg: str, log_err = False): for channel in bot.config.ip.warn_chans: @@ -165,88 +195,56 @@ def get_exemption(host): except: raise if not ip.is_global: - LOGGER.warn(f"Non-global IP {ip} seen.") + LOGGER.debug(f"Non-global IP {ip} seen.") return "Non-global IP; internal network, localhost, etc." for network, reason in exemptions.items(): if ip in network: return reason return None -def fetch_IPQS_ip_score( +def fetch_MaxMind_ip_score( ip_addr: typing.Union[ipaddress.IPv4Address, ipaddress.IPv6Address], key: str, - allow_public_access_points: bool = True, - lighter_penalties: bool = true, - strictness: int = 2, - fast: bool = False, - mobile: bool = False - ) -> tuple[float, bool, bool]: #score, is proxy, has recent abuse flag set + ) -> float: '''Perform lookup on a specific IP adress using ipqualityscore.com''' - params = urllib.parse.urlencode({ - 'allow_public_access_points': "true" if allow_public_access_points else "false", - 'strictness': int(strictness), - # lowercase + handle None and other non-bool garbage - 'fast': "true" if fast else "false", - 'mobile': "true" if mobile else "false", - 'lighter_penalties': "true" if lighter_penalties else "false", - }) - # ip_addr sourced from server, not user, so sanitization already done - with urllib.request.urlopen( - f"https://ipqualityscore.com/api/json/ip/{key}/{str(ip_addr)}?{params}") as url: - data = json.loads(url.read().decode()) + with MaxMind_lock: + data = client.score({'device': {'ip_address': ip_addr}}) LOGGER.debug(data) - if not data['success']: - errstr = f"{ip_addr} lookup failed with {data['message']}" - LOGGER.error(errstr) - raise RuntimeError(errstr) - return (data['fraud_score'], data["proxy"], data["recent_abuse"]) + final_score = max(float(data.risk_score), float(data.ip_address.risk)) + return final_score def get_ip_score_from_db(session, ip): query_result = session.query(KnownIPs)\ .filter(KnownIPs.ip == str(ip))\ .one_or_none() if query_result: - #Any known problematic provider should've been BADMAILed by now, but... - return (query_result.score, - query_result.flag_recent_abuse, - query_result.flag_is_proxy - ) + return query_result.score -def store_ip_score_in_db(session, ip, nick, IPQSresult): +def store_ip_score_in_db(session, ip, nick, MaxMind_score): new_known_ip = KnownIPs(ip= ip, - score= IPQSresult[0], - flag_recent_abuse= IPQSresult[1], - flag_is_proxy= IPQSresult[2]) + score= MaxMind_score) session.add(new_known_ip) session.commit() def retrieve_score(bot, ip, nick, do_fetch = True): - with ipqs_lock: - LOGGER.debug(f"Lock acquired. Beginning lookup for {str(ip)}") - global db_populated - session = bot.db.session() - try: - if not db_populated: - BASE.metadata.create_all(bot.db.engine) - db_populated = True - if retval := get_ip_score_from_db(session, ip): - return retval - elif do_fetch: - if IPQSresult := fetch_IPQS_ip_score(ip, bot.config.emailcheck.IPQS_key): - store_ip_score_in_db(session, ip, nick, IPQSresult) - return IPQSresult - else: #Shouldn't be possible - raise RuntimeError("Couldn't retrieve IPQS!") - else: - # If do_fetch is false, this is a best-effort request and shouldn't use up a query - return None - except SQLAlchemyError: - session.rollback() - raise + LOGGER.debug(f"Beginning lookup for {str(ip)}") + with sopel_session_scope(bot) as session: + if retval := get_ip_score_from_db(session, ip): + return retval + elif do_fetch: + if MaxMind_score := fetch_MaxMind_ip_score(ip, bot.config.ip.MaxMind_key): + store_ip_score_in_db(session, ip, nick, MaxMind_score) + return MaxMind_score + else: #Shouldn't be possible + raise RuntimeError("Couldn't retrieve IPQS!") + else: + # If do_fetch is false, this is a best-effort request and shouldn't use up a query + return None def _add_exemption(ip, reason): exemptions[ipaddress.ip_network(ip)] = reason +# Begin sopel code that I don't want to mess with def _decompress(source, target, delete_after_decompression=True): """Decompress just the database from the archive""" # https://stackoverflow.com/a/16452962 @@ -301,36 +299,46 @@ def _find_geoip_db(bot): return bot.config.core.homedir else: return False +# End Sopel code that I don't want to mess with def populate_user(bot, user, ip, host, nick): LOGGER.debug('Adding: %s!%s@%s with IP %s', nick, user, host, ip) user = bot.users.get(nick) or target.User(Identifier(nick), user, host) if ip: user.ip = ip # Add nonstandard field - bot.users[nick] = user # no-op if user was in users, needed otherwise + bot.users[Identifier(nick)] = user # no-op if user was in users, needed otherwise def zline(bot, ip, nick, duration): - if safe_mode: + if ip_safe_mode: LOGGER.info(f"SAFE MODE: Would zline {ip} for {duration}") else: bot.write("ZLINE", ip, duration, f":Auto z-line {nick}.") +def protect_chans(bot): + if ip_safe_mode: + LOGGER.info(f"SAFE MODE: Would protect chans") + return + for chan in bot.config.emailcheck.protect_chans: + bot.write("MODE", chan, "+R") + alert(bot, f"Setting {', '.join(bot.config.emailcheck.protect_chans)} +R") + def examine_user(bot, user, ip, host, nick): populate_user(bot, user, ip, host, nick) - res = retrieve_score(bot, ip, nick) - if res: - score, is_proxy, is_recent_abuse = res - if( is_proxy or is_recent_abuse or score >= bot.config.ip.malicious_threshold ): - alert(bot, f"{"Orps" if safe_mode else "Ops"}" - f": Nick {nick} has abuse score {score}, proxy: {is_proxy}, " - f"recent_abuse: {is_recent_abuse}; z-lining!") + if get_exemption(ip): + LOGGER.debug(f"Exempt IP {ip} for {nick}") + return + score = retrieve_score(bot, ip, nick) + if score is not None: + if( score >= bot.config.ip.malicious_threshold ): + alert(bot, ('Orps' if ip_safe_mode else 'Ops') + + f": Nick {nick} has abuse score {score}; z-lining!") duration = "24h" zline(bot, ip, nick, duration) protect_chans(bot) elif score >= bot.config.ip.warn_threshold: - alert(bot, f"{"Orps" if safe_mode else "Ops"}" + alert(bot, ('Orps' if ip_safe_mode else 'Ops') + f": Nick {nick} has abuse score {score}; keep an eye on them.") - return res + return score @module.event(events.RPL_WHOSPCRPL) @module.priority('high') @@ -345,9 +353,6 @@ def recv_whox_ip(bot, trigger): if len(trigger.args) != 6: return LOGGER.warning('While populating the IP DB, a WHO response was malformed.') _, _, user, ip, host, nick = trigger.args - if get_exemption(ip): - LOGGER.debug(f"Exempt IP {ip} for {nick}") - return examine_user(bot, user, ip, host, nick) @module.event(events.RPL_ENDOFWHO) @@ -357,6 +362,13 @@ def end_who_ip(bot, trigger): if 'WHOX' in bot.isupport: who_reqs.pop(trigger.args[1], None) +def get_whox_num(): + rand = str(randint(0, 999)) + while rand in who_reqs: + rand = str(randint(0, 999)) + who_reqs[rand] = True + return rand + @module.event(events.RPL_YOUREOPER) @module.priority('high') def send_who(bot, _): @@ -368,12 +380,9 @@ def send_who(bot, _): # 'x' indicates uncloaked address. This is triggered by # RPL_YOUREOPER because that functionality is restricted to opers. - rand = str(randint(0, 999)) - while rand in who_reqs: - rand = str(randint(0, 999)) + rand = get_whox_num() LOGGER.debug('Sending who: %s', rand) - who_reqs[rand] = True - bot.write([r'WHO * n%nuhti,' + rand]) + bot.write(['WHO * n%nuhti,' + rand]) #:safe.oh.us.irc.scpwiki.com NOTICE Kufat :*** CONNECT: Client connecting on port 6697 (class main): ASNbot!sopel@ool-45734b07.dyn.optonline.net (69.115.75.7) [Sopel: https://sopel.chat/] @module.rule(r'.*Client connecting .*: (\S*)!(\S*)@(\S*) \((.*)\)') @@ -382,7 +391,7 @@ def send_who(bot, _): def handle_snotice_conn(bot, trigger): LOGGER.debug("Saw connect line: [%s] from [%s]", trigger.raw, trigger.sender) #Only servers may have '.' in the sender name, so this isn't spoofable - if "scpwiki.com" in trigger.sender: + if bot.config.ip.sno_string in trigger.sender: nick, user, host, ip = trigger.groups() # We need to check if the IP is in any exempt CIDR ranges if get_exemption(ip): @@ -391,13 +400,11 @@ def handle_snotice_conn(bot, trigger): if any(host.endswith(s) for s in (".irccloud.com", ".mibbit.com")): LOGGER.error(f"{host} slipped through get_exemption()") return - res = examine_user(bot, user, ip, host, nick) - if res: + score = examine_user(bot, user, ip, host, nick) + if score: # Acted on above; just log here - score, is_proxy, is_recent_abuse = res LOGGER.debug(f"handle_snotice_conn: {nick}!{user}@{host} ({ip}) had " - "score {score}, proxy: {is_prox}, " - "recent_abuse: {is_recent_abuse}") + f"score {score}") # NICK: User Kufat-bar changed their nickname to Kufat-foo # This is redundant for users the bot can see in-channel but needed for users with no common channel @@ -406,7 +413,7 @@ def handle_snotice_conn(bot, trigger): @module.priority("high") def handle_snotice_ren(bot, trigger): LOGGER.debug("Saw nick change line: [%s] from [%s]", trigger.raw, trigger.sender) - if "scpwiki.com" in trigger.sender: + if bot.config.ip.sno_string in trigger.sender: oldnick = Identifier(trigger.group(1)) newnick = Identifier(trigger.group(2)) if olduser := bot.users.get(oldnick): @@ -415,9 +422,9 @@ def handle_snotice_ren(bot, trigger): @module.require_owner @module.commands('toggle_safe_ip') def toggle_safe(bot, trigger): - global safe_mode - safe_mode = not safe_mode - return bot.reply(f"IP module safe mode now {'on' if safe_mode else 'off'}") + global ip_safe_mode + ip_safe_mode = not ip_safe_mode + return bot.reply(f"IP module safe mode now {'on' if ip_safe_mode else 'off'}") @module.require_privilege(module.OP) @module.commands('ip_exempt') @@ -446,10 +453,10 @@ def ip_exempt(bot, trigger): ignore='Downloading GeoIP database, please wait...', online=True) def ip(bot, trigger): - if trigger.is_privmsg and ( trigger.account is None or trigger.account.lower() != "kufat" ): - return + if trigger.is_privmsg and not trigger.admin: + return bot.reply("You're not my supervisor!") full = ( ( trigger.sender.lower() in ("#skipirc-staff", "#kufat") ) or - ( trigger.is_privmsg and trigger.account.lower() == "kufat" ) ) + trigger.is_privmsg) irccloud = False mibbit = False nick = None @@ -477,11 +484,13 @@ def ip(bot, trigger): #TODO TODO TODO get from DB return bot.say("I\'m not aware of this user.") - ex = get_exemption(query).lower() + ex = get_exemption(query) if ex: - irccloud = "irccloud" in ex - mibbit = "mibbit" in ex + exl = ex.lower() + irccloud = "irccloud" in exl + mibbit = "mibbit" in exl + if not any((irccloud, mibbit)): db_path = _find_geoip_db(bot) if db_path is False: @@ -522,7 +531,7 @@ def ip(bot, trigger): elif mibbit: response += " IP belongs to mibbit; no location data available" return bot.say(response) - else: + elif ex: response += f" | IP meets exemption [{ex}] |" # Still look up an IP that's exempt for other reasons @@ -551,13 +560,13 @@ def ip(bot, trigger): response += ' ISP: Unknown' force_lookup = trigger.group(4) == "lookup" - res = None + score = None try: - res = retrieve_score(bot, query, nick, force_lookup) + score = retrieve_score(bot, query, nick, force_lookup) except Exception as e: LOGGER.error(f"Couldn't look up IP {query} because {e}") - if res: - response += f" | Score: {res[0]} Proxy: {res[1]} Recent abuse detected: {res[2]}" + if score is not None: + response += f" | Score: {score}" elif not force_lookup: # Use search_str to avoid leaking an IP response += f" | To retrieve IP score run '.ip {search_str} lookup'" From 6d9397850efdbe709a6a6c23b2f8e047ad65a7df Mon Sep 17 00:00:00 2001 From: Kufat Date: Sun, 31 Jan 2021 22:58:10 -0500 Subject: [PATCH 08/11] Bug fixes. TODO get rid of duplicate dataclass/named tuple. --- sopel/modules/emailcheck.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sopel/modules/emailcheck.py b/sopel/modules/emailcheck.py index 16b5305139..86baca3f1b 100644 --- a/sopel/modules/emailcheck.py +++ b/sopel/modules/emailcheck.py @@ -90,8 +90,8 @@ def __post_init__(self): @dataclass class DomainInfo: - flag_disposable: bool flag_valid: bool + flag_disposable: bool def alert(bot, alert_msg: str, log_err: bool = False): for channel in bot.config.emailcheck.warn_chans: @@ -218,7 +218,7 @@ def fetch_validator_pizza_email_info(email_addr: str ) \ return ValidatorPizzaResponse(data['mx'], data["disposable"]) elif data['status'] == HTTPStatus.BAD_REQUEST: # Address is invalid, assume typo - return (False, None) + return ValidatorPizzaResponse(False, None) elif data['status'] == HTTPStatus.TOO_MANY_REQUESTS: # This is unlikely enough that I'm going to postpone dealing with it raise RuntimeError("Hit request limit!") From f4e825fc98b1e2a2053b0e377d029eaf3da93b8f Mon Sep 17 00:00:00 2001 From: Kufat Date: Sun, 31 Jan 2021 23:00:21 -0500 Subject: [PATCH 09/11] Don't MSG users when safe mode is on --- sopel/modules/emailcheck.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/sopel/modules/emailcheck.py b/sopel/modules/emailcheck.py index 86baca3f1b..4919aeede4 100644 --- a/sopel/modules/emailcheck.py +++ b/sopel/modules/emailcheck.py @@ -178,10 +178,11 @@ def protect_chans(bot): def malicious_response(bot, nick: str, email): fdrop(bot, nick) add_badmail(bot, email) - bot.say(f"You have been temporarily banned from this network because {email.domain} " - "has a history of spam or abuse, and/or is a disposable email domain. " - "If this is a legitimate domain, contact staff for assistance.", - nick.lower()) + if not email_safe_mode: + bot.say(f"You have been temporarily banned from this network because {email.domain} " + "has a history of spam or abuse, and/or is a disposable email domain. " + "If this is a legitimate domain, contact staff for assistance.", + nick.lower()) gline_or_kill(bot, nick, bot.config.emailcheck.gline_time) protect_chans(bot) alert(bot, f"ALERT: User {nick} attempted to register a nick with disposable/spam domain {email.domain}!") @@ -189,9 +190,10 @@ def malicious_response(bot, nick: str, email): def disallow_response(bot, nick: str, email): fdrop(bot, nick) add_badmail(bot, email) - bot.say(f"Your registration has been disallowed because {email.domain} appears to be suspicious. " - "If this is a legitimate domain, contact staff for assistance.", - nick.lower()) + if not email_safe_mode: + bot.say(f"Your registration has been disallowed because {email.domain} appears to be suspicious. " + "If this is a legitimate domain, contact staff for assistance.", + nick.lower()) alert(bot, f"WARNING: User {nick} attempted to register a nick with suspicious domain {email.domain}.") def fetch_validator_pizza_email_info(email_addr: str ) \ From cd1a28013cc0668132351b16d867aa796ffb58d5 Mon Sep 17 00:00:00 2001 From: Kufat Date: Mon, 1 Feb 2021 11:37:52 -0500 Subject: [PATCH 10/11] Remove redundant DomainInfo dataclass --- sopel/modules/emailcheck.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/sopel/modules/emailcheck.py b/sopel/modules/emailcheck.py index 4919aeede4..980a34b236 100644 --- a/sopel/modules/emailcheck.py +++ b/sopel/modules/emailcheck.py @@ -88,11 +88,6 @@ def __str__(self): def __post_init__(self): self.domain = self.domain.lower() -@dataclass -class DomainInfo: - flag_valid: bool - flag_disposable: bool - def alert(bot, alert_msg: str, log_err: bool = False): for channel in bot.config.emailcheck.warn_chans: bot.say(alert_msg, channel) @@ -235,14 +230,14 @@ def get_email_info_from_db(session, email): .one_or_none() if query_result: #Any known problematic provider should've been BADMAILed by now, but... - return DomainInfo(query_result.flag_valid, - query_result.flag_disposable) + return ValidatorPizzaResponse(flag_valid=query_result.flag_valid, + flag_disposable=query_result.flag_disposable) def store_email_info_in_db(session, email, nick, result): - new_known_email = KnownEmails(domain= email.domain[:DOMAIN_LEN], - first_nick= nick, - flag_valid= result.flag_valid, - flag_disposable= result.flag_disposable) + new_known_email = KnownEmails(domain=email.domain[:DOMAIN_LEN], + first_nick=nick, + flag_valid=result.flag_valid, + flag_disposable=result.flag_disposable) session.add(new_known_email) def retrieve_info_for_email(bot, email, nick): From ebd63d8b29d560b223127b09165a3ec473d0dfed Mon Sep 17 00:00:00 2001 From: Kufat Date: Tue, 1 Jun 2021 21:45:47 -0400 Subject: [PATCH 11/11] Current state of Kufat's local repo --- sopel/modules/emailcheck.py | 31 ++++++++++---------- sopel/modules/ip.py | 56 ++++++++++++++++++++++--------------- 2 files changed, 49 insertions(+), 38 deletions(-) diff --git a/sopel/modules/emailcheck.py b/sopel/modules/emailcheck.py index 980a34b236..060d77d5d6 100644 --- a/sopel/modules/emailcheck.py +++ b/sopel/modules/emailcheck.py @@ -32,7 +32,7 @@ IRCCLOUD_USER_REGEX = re.compile(r"[us]id[0-9]{4,}") DOMAIN_LEN = 50 -KILL_STR = ":Use of disposable email service for nick registration" +KILL_STR = "Use of disposable email service for nick registration" LOGGER = logging.getLogger(__name__) @@ -45,7 +45,7 @@ ValidatorPizzaResponse = namedtuple('ValidatorPizzaResponse', ['flag_valid', 'flag_disposable']) -GLineStrategy = namedtuple('GLineStrategy', ['strategy', 'targer']) +GLineStrategy = namedtuple('GLineStrategy', ['strategy', 'target']) #SQLAlchemy container class class KnownEmails(BASE): @@ -99,19 +99,19 @@ def add_badmail(bot, email): if email_safe_mode: LOGGER.info(f"SAFE MODE: Would badmail {email}") else: - bot.write("NICKSERV", "badmail", "add", f'*@{email.domain}') + bot.write(("NICKSERV", "badmail", "add", f'*@{email.domain}'), "Disposable email domain") def fdrop(bot, nick: str): if email_safe_mode: LOGGER.info(f"SAFE MODE: Would fdrop {nick}") else: - bot.write("NICKSERV", "fdrop", nick.lower()) + bot.write(("NICKSERV", "fdrop", nick.lower())) def gline_ip(bot, ip: str, duration: str): if email_safe_mode: LOGGER.info(f"SAFE MODE: Would gline {ip} for {duration}") else: - bot.write("GLINE", f'*@{ip}', duration, KILL_STR) + bot.write(("GLINE", f'*@{ip}', duration), KILL_STR) def gline_irccloud(bot, nick: str, duration: str): if known_user := bot.users.get(Identifier(nick)): @@ -120,7 +120,7 @@ def gline_irccloud(bot, nick: str, duration: str): if email_safe_mode: LOGGER.info(f"SAFE MODE: Would gline {username} for {duration}") else: - bot.write("GLINE", f'{username}@*', duration, KILL_STR) + bot.write(("GLINE", f'{username}@*', duration), KILL_STR) return else: alert(bot, f"User {nick} had unexpected non-IRCCloud username {username}", true) @@ -132,7 +132,7 @@ def kill_nick(bot, nick: str): if email_safe_mode: LOGGER.info(f"SAFE MODE: Would kill {nick}") else: - bot.write("KILL", nick.lower(), KILL_STR) + bot.write(("KILL", nick.lower()), KILL_STR) def gline_strategy(bot, nick): if (known_user := bot.users.get(Identifier(nick))): @@ -153,11 +153,11 @@ def gline_strategy(bot, nick): def gline_or_kill(bot, nick: str, duration: str): if gline_strat := gline_strategy(bot, nick): if gline_strat.strategy == "gline_ip": - gline_ip(bot, strategy.target, duration) + gline_ip(bot, gline_strat.target, duration) elif gline_strat.strategy == "gline_irccloud": - gline_irccloud(bot, strategy.target, duration) + gline_irccloud(bot, gline_strat.target, duration) else: - alert(bot, f"Unknown strategy {strategy} for nick {nick}", true) + alert(bot, f"Unknown strategy {gline_strat.strategy} for nick {nick}", true) kill_nick(bot, nick) # safest option else: kill_nick(bot, nick) # duration ignored @@ -167,8 +167,9 @@ def protect_chans(bot): LOGGER.info(f"SAFE MODE: Would protect chans") return for chan in bot.config.emailcheck.protect_chans: - bot.write("MODE", chan, "+R") - alert(bot, f"Setting {', '.join(bot.config.emailcheck.protect_chans)} +R") + bot.write(("MODE", chan, "+R")) + if len(bot.config.emailcheck.protect_chans) > 0: + alert(bot, f"Setting {', '.join(bot.config.emailcheck.protect_chans)} +R") def malicious_response(bot, nick: str, email): fdrop(bot, nick) @@ -256,9 +257,9 @@ def retrieve_info_for_email(bot, email, nick): @module.require_owner @module.commands('toggle_safe_email') def toggle_safe(bot, trigger): - global safe_mode - safe_mode = not safe_mode - return bot.reply(f"Email check module safe mode now {'ON' if safe_mode else 'OFF'}") + global email_safe_mode + email_safe_mode = not email_safe_mode + return bot.reply(f"Email check module safe mode now {'ON' if email_safe_mode else 'OFF'}") # ExampleAccount REGISTER: ExampleNick to foo@example.com # (note the 0x02 bold chars) diff --git a/sopel/modules/ip.py b/sopel/modules/ip.py index c8c6286875..b659f66a58 100644 --- a/sopel/modules/ip.py +++ b/sopel/modules/ip.py @@ -8,8 +8,6 @@ https://sopel.chat """ -from __future__ import unicode_literals, absolute_import, print_function, division - import ipaddress import logging import os @@ -80,9 +78,16 @@ ("2604:a880:2:d0::/64", "noracodes"), ("2a01:4f8:202:62c8::/64", "grumble"), ("78.46.73.141", "hooloovoo"), + ("2a01:4f8:120:4091::/64", "hooloovoo"), ("67.205.43.220", "carolynn ivy, dreamhost"), ("212.47.230.56", "skee, token.ro"), ("2600:3c01::/64", "atomicthumbs"), + ("3.13.93.249", "Lounge"), + ("44.135.218.31", "Kufat's shell"), + ("159.203.5.135", "BytesAndCoffee's ZNC"), + ("5.9.158.70", "john@soupwhale.com, usual nick 'john'"), + ("24.85.200.227", "Guildlight; seems to be a legit user."), + ("120.28.217.130", "GeoRrey; normal user with tech issues, can't afford to replace vulnerable equipment") ] LOGGER = logging.getLogger(__name__) @@ -104,6 +109,9 @@ class KnownIPs(BASE): __tablename__ = 'known_ips' ip = Column(String(IP_MAX_LEN), primary_key=True, unique=True, index=True) + # For IPv4: address as 32-bit int. For IPv6: /64 network right-shifted 64-bits. + # 0::/8 is reserved, so no collisions. + # ip_int_representation = Column(Integer, index=True, nullable=False) score = Column(Float, nullable=False) insert_time = Column(TIMESTAMP, server_default=sqlalchemy.sql.func.now()) @@ -193,6 +201,7 @@ def get_exemption(host): try: ip = ipaddress.ip_address(socket.getaddrinfo(host, None)[0][4][0]) except: + LOGGER.error(f"Couldn't get IP for host {host}") raise if not ip.is_global: LOGGER.debug(f"Non-global IP {ip} seen.") @@ -209,18 +218,18 @@ def fetch_MaxMind_ip_score( '''Perform lookup on a specific IP adress using ipqualityscore.com''' with MaxMind_lock: data = client.score({'device': {'ip_address': ip_addr}}) - LOGGER.debug(data) + LOGGER.debug(f"Result from MaxMind for {ip_addr}: {data}") final_score = max(float(data.risk_score), float(data.ip_address.risk)) return final_score -def get_ip_score_from_db(session, ip): +def get_ip_score_from_db(session, ip, ipv6_network): query_result = session.query(KnownIPs)\ .filter(KnownIPs.ip == str(ip))\ .one_or_none() if query_result: return query_result.score -def store_ip_score_in_db(session, ip, nick, MaxMind_score): +def store_ip_score_in_db(session, ip, ipv6_network, nick, MaxMind_score): new_known_ip = KnownIPs(ip= ip, score= MaxMind_score) session.add(new_known_ip) @@ -228,12 +237,19 @@ def store_ip_score_in_db(session, ip, nick, MaxMind_score): def retrieve_score(bot, ip, nick, do_fetch = True): LOGGER.debug(f"Beginning lookup for {str(ip)}") + ipv6_network = None + try: + ipv6 = ipaddress.IPv6Address(ip) + ipv6_network = int(ipv6) >> 64 + except ipaddress.AddressValueError: + pass with sopel_session_scope(bot) as session: - if retval := get_ip_score_from_db(session, ip): + if retval := get_ip_score_from_db(session, ip, ipv6_network): + LOGGER.debug(f"{str(ip)} score retrieved from DB: {retval}") return retval elif do_fetch: if MaxMind_score := fetch_MaxMind_ip_score(ip, bot.config.ip.MaxMind_key): - store_ip_score_in_db(session, ip, nick, MaxMind_score) + store_ip_score_in_db(session, ip, ipv6_network, nick, MaxMind_score) return MaxMind_score else: #Shouldn't be possible raise RuntimeError("Couldn't retrieve IPQS!") @@ -312,15 +328,16 @@ def zline(bot, ip, nick, duration): if ip_safe_mode: LOGGER.info(f"SAFE MODE: Would zline {ip} for {duration}") else: - bot.write("ZLINE", ip, duration, f":Auto z-line {nick}.") + bot.write(("ZLINE", ip, duration), f":Auto z-line {nick}.") def protect_chans(bot): if ip_safe_mode: LOGGER.info(f"SAFE MODE: Would protect chans") return for chan in bot.config.emailcheck.protect_chans: - bot.write("MODE", chan, "+R") - alert(bot, f"Setting {', '.join(bot.config.emailcheck.protect_chans)} +R") + bot.write(("MODE", chan, "+R")) + if len(bot.config.emailcheck.protect_chans) > 0: + alert(bot, f"Setting {', '.join(bot.config.emailcheck.protect_chans)} +R") def examine_user(bot, user, ip, host, nick): populate_user(bot, user, ip, host, nick) @@ -336,8 +353,7 @@ def examine_user(bot, user, ip, host, nick): zline(bot, ip, nick, duration) protect_chans(bot) elif score >= bot.config.ip.warn_threshold: - alert(bot, ('Orps' if ip_safe_mode else 'Ops') + - f": Nick {nick} has abuse score {score}; keep an eye on them.") + alert(bot, f"Attention: Nick {nick} has abuse score {score}; keep an eye on them.") return score @module.event(events.RPL_WHOSPCRPL) @@ -385,7 +401,7 @@ def send_who(bot, _): bot.write(['WHO * n%nuhti,' + rand]) #:safe.oh.us.irc.scpwiki.com NOTICE Kufat :*** CONNECT: Client connecting on port 6697 (class main): ASNbot!sopel@ool-45734b07.dyn.optonline.net (69.115.75.7) [Sopel: https://sopel.chat/] -@module.rule(r'.*Client connecting .*: (\S*)!(\S*)@(\S*) \((.*)\)') +@module.rule(r'.*Client connecting .*: (\S*)!(\S*)@(\S*) \((\S+)\)') @module.event("NOTICE") @module.priority("high") def handle_snotice_conn(bot, trigger): @@ -393,13 +409,6 @@ def handle_snotice_conn(bot, trigger): #Only servers may have '.' in the sender name, so this isn't spoofable if bot.config.ip.sno_string in trigger.sender: nick, user, host, ip = trigger.groups() - # We need to check if the IP is in any exempt CIDR ranges - if get_exemption(ip): - return - #Be **certain** we don't waste our lookups on irccloud - if any(host.endswith(s) for s in (".irccloud.com", ".mibbit.com")): - LOGGER.error(f"{host} slipped through get_exemption()") - return score = examine_user(bot, user, ip, host, nick) if score: # Acted on above; just log here @@ -419,7 +428,7 @@ def handle_snotice_ren(bot, trigger): if olduser := bot.users.get(oldnick): populate_user(bot, olduser.user, olduser.ip, olduser.host, newnick) -@module.require_owner +@module.require_admin @module.commands('toggle_safe_ip') def toggle_safe(bot, trigger): global ip_safe_mode @@ -427,7 +436,7 @@ def toggle_safe(bot, trigger): return bot.reply(f"IP module safe mode now {'on' if ip_safe_mode else 'off'}") @module.require_privilege(module.OP) -@module.commands('ip_exempt') +@module.commands('ip_exempt', 'add_exemption') @module.example('.ip_exempt 8.8.8.8 Known user example123\'s bouncer') def ip_exempt(bot, trigger): ipstr = trigger.group(3) # arg 1 @@ -453,6 +462,7 @@ def ip_exempt(bot, trigger): ignore='Downloading GeoIP database, please wait...', online=True) def ip(bot, trigger): + LOGGER.debug(trigger) if trigger.is_privmsg and not trigger.admin: return bot.reply("You're not my supervisor!") full = ( ( trigger.sender.lower() in ("#skipirc-staff", "#kufat") ) or @@ -532,7 +542,7 @@ def ip(bot, trigger): response += " IP belongs to mibbit; no location data available" return bot.say(response) elif ex: - response += f" | IP meets exemption [{ex}] |" + response += f" IP meets exemption [{ex}] |" # Still look up an IP that's exempt for other reasons if full: