From d731fe9f91562a7d69bfcd6d272830c0cf18507b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20K=C3=BCthe?= Date: Sun, 22 Oct 2023 18:37:40 +0200 Subject: [PATCH 1/2] Add I2P Support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merge https://github.com/HelloZeroNet/ZeroNet/pull/602 Signed-off-by: Marek Küthe --- README.md | 6 +- Vagrantfile | 2 +- requirements.txt | 1 + src/Config.py | 5 + src/Connection/Connection.py | 51 ++++++--- src/Connection/ConnectionServer.py | 15 ++- src/File/FileRequest.py | 12 +- src/File/FileServer.py | 1 + src/I2P/I2PManager.py | 176 +++++++++++++++++++++++++++++ src/I2P/__init__.py | 1 + src/Peer/Peer.py | 14 ++- src/Site/Site.py | 19 ++++ src/Site/SiteAnnouncer.py | 5 + src/Test/TestI2P.py | 135 ++++++++++++++++++++++ src/Test/conftest.py | 12 ++ src/Ui/UiWebsocket.py | 6 +- src/util/helper.py | 13 ++- 17 files changed, 447 insertions(+), 27 deletions(-) create mode 100644 src/I2P/I2PManager.py create mode 100644 src/I2P/__init__.py create mode 100644 src/Test/TestI2P.py diff --git a/README.md b/README.md index 70b79adca..8a299f8aa 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,9 @@ Decentralized websites using Bitcoin crypto and the BitTorrent network - https:/ * Password-less [BIP32](https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki) based authorization: Your account is protected by the same cryptography as your Bitcoin wallet * Built-in SQL server with P2P data synchronization: Allows easier site development and faster page load times - * Anonymity: Full Tor network support with .onion hidden services instead of IPv4 addresses + * Anonymity: + * Full Tor network support with .onion hidden services instead of IPv4 addresses + * Full I2P network support with I2P Destinations instead of IPv4 addresses * TLS encrypted connections * Automatic uPnP port opening * Plugin for multiuser (openproxy) support @@ -132,7 +134,7 @@ https://zeronet.ipfsscan.io/ * File transactions are not compressed * No private sites - +* ~~No more anonymous than Bittorrent~~ (built-in full Tor and I2P support added) ## How can I create a ZeroNet site? diff --git a/Vagrantfile b/Vagrantfile index 24fe0c45f..10a11c58a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -40,6 +40,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.provision "shell", inline: "sudo apt-get install msgpack-python python-gevent python-pip python-dev -y" config.vm.provision "shell", - inline: "sudo pip install msgpack --upgrade" + inline: "sudo pip install -r requirements.txt --upgrade" end diff --git a/requirements.txt b/requirements.txt index 538a6dfcd..449acc18a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ websocket_client gevent-ws coincurve maxminddb +i2p.socket \ No newline at end of file diff --git a/src/Config.py b/src/Config.py index a9208d556..2c124ff96 100644 --- a/src/Config.py +++ b/src/Config.py @@ -88,6 +88,8 @@ def createArguments(self): "http://t.publictracker.xyz:6969/announce", "https://tracker.lilithraws.cf:443/announce", "https://tracker.babico.name.tr:443/announce", + "http://opentracker.dg2.i2p/announce", + "http://opentracker.skank.i2p/announce" ] # Platform specific if sys.platform.startswith("win"): @@ -311,6 +313,9 @@ def createArguments(self): self.parser.add_argument('--tor_use_bridges', help='Use obfuscated bridge relays to avoid Tor block', action='store_true') self.parser.add_argument('--tor_hs_limit', help='Maximum number of hidden services in Tor always mode', metavar='limit', type=int, default=10) self.parser.add_argument('--tor_hs_port', help='Hidden service port in Tor always mode', metavar='limit', type=int, default=15441) + + self.parser.add_argument('--i2p', help='enable: Use only for I2P peers, always: Use I2P for every connection', choices=["disable", "enable", "always"], default='enable') + self.parser.add_argument('--i2p_sam', help='I2P SAM API address', metavar='ip:port', default='127.0.0.1:7656') self.parser.add_argument('--version', action='version', version='ZeroNet %s r%s' % (self.version, self.rev)) self.parser.add_argument('--end', help='Stop multi value argument parsing', action='store_true') diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py index 22bcf29cd..12fcd870c 100644 --- a/src/Connection/Connection.py +++ b/src/Connection/Connection.py @@ -133,6 +133,10 @@ def connect(self): self.sock = socks.socksocket() proxy_ip, proxy_port = config.trackers_proxy.split(":") self.sock.set_proxy(socks.PROXY_TYPE_SOCKS5, proxy_ip, int(proxy_port)) + elif self.ip.endswith(".i2p"): + if not self.server.i2p_manager or not self.server.i2p_manager.enabled: + raise Exception("Can't connect to I2P addresses, no SAM API present") + self.sock = self.server.i2p_manager.createSocket(self.ip, self.port) else: self.sock = self.createSocket() @@ -344,22 +348,27 @@ def handleStream(self, message, buff): # My handshake info def getHandshakeInfo(self): # No TLS for onion connections - if self.ip_type == "onion": + if self.ip_type == "onion" or self.ip_type == "i2p": crypt_supported = [] elif self.ip in self.server.broken_ssl_ips: crypt_supported = [] else: crypt_supported = CryptConnection.manager.crypt_supported # No peer id for onion connections - if self.ip_type == "onion" or self.ip in config.ip_local: + if self.ip_type == "onion" or self.ip_type == "i2p" or self.ip in config.ip_local: peer_id = "" else: peer_id = self.server.peer_id # Setup peer lock from requested onion address - if self.handshake and self.handshake.get("target_ip", "").endswith(".onion") and self.server.tor_manager.start_onions: - self.target_onion = self.handshake.get("target_ip").replace(".onion", "") # My onion address - if not self.server.tor_manager.site_onions.values(): - self.server.log.warning("Unknown target onion address: %s" % self.target_onion) + if self.handshake: + if self.handshake.get("target_ip", "").endswith(".onion") and self.server.tor_manager.start_onions: + self.target_onion = self.handshake.get("target_ip").replace(".onion", "") # My onion address + if not self.server.tor_manager.site_onions.values(): + self.server.log.warning("Unknown target onion address: %s" % self.target_onion) + elif self.handshake.get("target_ip", "").endswith(".i2p") and self.server.i2p_manager.start_dests: + self.target_dest = self.handshake.get("target_ip").replace(".i2p", "") # My I2P Destination + if not dest_sites.get(target_dest): + self.server.log.error("Unknown target I2P Destination: %s" % target_dest) handshake = { "version": config.version, @@ -378,6 +387,10 @@ def getHandshakeInfo(self): handshake["onion"] = self.target_onion elif self.ip_type == "onion": handshake["onion"] = self.server.tor_manager.getOnion("global") + elif self.target_dest: + handshake["i2p"] = self.target_dest + elif self.ip_type == "i2p": + handshake["i2p"] = self.server.i2p_manager.getDest("global").base64() if self.is_tracker_connection: handshake["tracker_connection"] = True @@ -397,7 +410,7 @@ def setHandshake(self, handshake): return False self.handshake = handshake - if handshake.get("port_opened", None) is False and "onion" not in handshake and not self.is_private_ip: # Not connectable + if handshake.get("port_opened", None) is False and "onion" not in handshake and "i2p" not in handshake and not self.is_private_ip: # Not connectable self.port = 0 else: self.port = int(handshake["fileserver_port"]) # Set peer fileserver port @@ -416,7 +429,7 @@ def setHandshake(self, handshake): if type(handshake["crypt_supported"][0]) is bytes: handshake["crypt_supported"] = [item.decode() for item in handshake["crypt_supported"]] # Backward compatibility - if self.ip_type == "onion" or self.ip in config.ip_local: + if self.ip_type == "onion" or self.ip_type == "i2p" or self.ip in config.ip_local: crypt = None elif handshake.get("crypt"): # Recommended crypt by server crypt = handshake["crypt"] @@ -426,13 +439,21 @@ def setHandshake(self, handshake): if crypt: self.crypt = crypt - if self.type == "in" and handshake.get("onion") and not self.ip_type == "onion": # Set incoming connection's onion address - if self.server.ips.get(self.ip) == self: - del self.server.ips[self.ip] - self.setIp(handshake["onion"] + ".onion") - self.log("Changing ip to %s" % self.ip) - self.server.ips[self.ip] = self - self.updateName() + if self.type == "in": + if handshake.get("onion") and not self.ip_type == "onion": # Set incoming connection's onion address + if self.server.ips.get(self.ip) == self: + del self.server.ips[self.ip] + self.setIp(handshake["onion"] + ".onion") + self.log("Changing ip to %s" % self.ip) + self.server.ips[self.ip] = self + self.updateName() + if handshake.get("i2p") and not self.ip_type == "i2p": # Set incoming connection's I2P Destination + if self.server.ips.get(self.ip) == self: + del self.server.ips[self.ip] + self.setIp(handshake["i2p"] + ".i2p") + self.log("Changing ip to %s" % self.ip) + self.server.ips[self.ip] = self + self.updateName() self.event_connected.set(True) # Mark handshake as done self.event_connected = None diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index c90483988..b3a3f6b4a 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -17,6 +17,7 @@ from Crypt import CryptConnection from Crypt import CryptHash from Tor import TorManager +from I2P import I2PManager from Site import SiteManager @@ -38,6 +39,10 @@ def __init__(self, ip=None, port=None, request_handler=None): self.peer_blacklist = SiteManager.peer_blacklist self.tor_manager = TorManager(self.ip, self.port) + if config.i2p != "disabled": + self.i2p_manager = I2PManager(self.handleIncomingConnection) + else: + self.i2p_manager = None self.connections = [] # Connections self.whitelist = config.ip_local # No flood protection on this ips self.ip_incoming = {} # Incoming connections from ip in the last minute to avoid connection flood @@ -171,10 +176,13 @@ def handleMessage(self, *args, **kwargs): def getConnection(self, ip=None, port=None, peer_id=None, create=True, site=None, is_tracker_connection=False): ip_type = helper.getIpType(ip) - has_per_site_onion = (ip.endswith(".onion") or self.port_opened.get(ip_type, None) == False) and self.tor_manager.start_onions and site - if has_per_site_onion: # Site-unique connection for Tor + has_per_site_onion = (((ip.endswith(".onion") or self.port_opened.get("onion", None) == False) and self.tor_manager.start_onions) or \ + ((ip.endswith(".i2p") or self.port_opened.get("i2p", None) == False) and self.i2p_manager.start_dests)) and site + if has_per_site_onion: # Site-unique connection for Tor or I2P if ip.endswith(".onion"): site_onion = self.tor_manager.getOnion(site.address) + elif ip.endswith(".i2p"): + site_onion = self.i2p_manager.getDest(site.address) else: site_onion = self.tor_manager.getOnion("global") key = ip + site_onion @@ -196,7 +204,8 @@ def getConnection(self, ip=None, port=None, peer_id=None, create=True, site=None if connection.ip == ip: if peer_id and connection.handshake.get("peer_id") != peer_id: # Does not match continue - if ip.endswith(".onion") and self.tor_manager.start_onions and ip.replace(".onion", "") != connection.target_onion: + if (ip.endswith(".onion") and self.tor_manager.start_onions and ip.replace(".onion", "") != connection.target_onion) or \ + (ip.endswith(".i2p") and self.i2p_manager.start_dests and ip.replace(".i2p", "") != connection.target_dest): # For different site continue if not connection.connected and create: diff --git a/src/File/FileRequest.py b/src/File/FileRequest.py index c082c378d..9a0b3fd85 100644 --- a/src/File/FileRequest.py +++ b/src/File/FileRequest.py @@ -321,6 +321,13 @@ def actionPex(self, params): if site.addPeer(*address, source="pex"): added += 1 + # Add sent i2p peers to site + for packed_address in params.get("peers_i2p", []): + address = helper.unpackI2PAddress(packed_address) + got_peer_keys.append("%s:%s" % address) + if site.addPeer(*address): + added += 1 + # Send back peers that is not in the sent list and connectable (not port 0) packed_peers = helper.packPeers(site.getConnectablePeers(params["need"], ignore=got_peer_keys, allow_private=False)) @@ -335,7 +342,8 @@ def actionPex(self, params): back = { "peers": packed_peers["ipv4"], "peers_ipv6": packed_peers["ipv6"], - "peers_onion": packed_peers["onion"] + "peers_onion": packed_peers["onion"], + "peers_i2p": packed_peers["i2p"] } self.response(back) @@ -410,7 +418,7 @@ def actionFindHashIds(self, params): "Found: %s for %s hashids in %.3fs" % ({key: len(val) for key, val in back.items()}, len(params["hash_ids"]), time.time() - s) ) - self.response({"peers": back["ipv4"], "peers_onion": back["onion"], "peers_ipv6": back["ipv6"], "my": my_hashes}) + self.response({"peers": back["ipv4"], "peers_onion": back["onion"], "peers_i2p": back["i2p"], "peers_ipv6": back["ipv6"], "my": my_hashes}) def actionSetHashfield(self, params): site = self.sites.get(params["site"]) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index b7a942fc5..899db8b51 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -252,6 +252,7 @@ def checkSites(self, check_files=False, force_port_check=False): if not self.port_opened["ipv4"]: self.tor_manager.startOnions() + self.i2p_manager.startDests() if not sites_checking: check_pool = gevent.pool.Pool(5) diff --git a/src/I2P/I2PManager.py b/src/I2P/I2PManager.py new file mode 100644 index 000000000..51fb86333 --- /dev/null +++ b/src/I2P/I2PManager.py @@ -0,0 +1,176 @@ +import logging + +from gevent.lock import RLock +from gevent.server import StreamServer +from gevent.pool import Pool +from http.client import HTTPConnection +import urllib.request, urllib.error, urllib.parse + +from i2p import socket +from i2p.datatypes import Destination + +from Config import config +from Site import SiteManager +from Debug import Debug + + +class I2PHTTPConnection(HTTPConnection): + def __init__(self, i2p_manager, site_address, *args, **kwargs): + HTTPConnection.__init__(self, *args, **kwargs) + self.i2p_manager = i2p_manager + self.site_address = site_address + self._create_connection = self._create_i2p_connection + + def _create_i2p_connection(self, address, timeout=60, + source_address=None): + return self.i2p_manager.createSocket(self.site_address, *address) + +class I2PHTTPHandler(urllib.request.HTTPHandler): + def __init__(self, i2p_manager, site_address, *args, **kwargs): + urllib.request.HTTPHandler.__init__(self, *args, **kwargs) + self.i2p_manager = i2p_manager + self.site_address = site_address + + def http_open(self, req): + return self.do_open(self._createI2PHTTPConnection, req) + + def _createI2PHTTPConnection(self, *args, **kwargs): + return I2PHTTPConnection(self.i2p_manager, self.site_address, *args, **kwargs) + +class I2PManager: + def __init__(self, fileserver_handler=None): + self.dest_conns = {} # Destination: SAM connection + self.dest_servs = {} # Destination: StreamServer + self.site_dests = {} # Site address: Destination + self.log = logging.getLogger("I2PManager") + self.start_dests = None + self.lock = RLock() + + if config.i2p == "disable": + self.enabled = False + self.start_dests = False + self.status = "Disabled" + else: + self.enabled = True + self.status = "Waiting" + + if fileserver_handler: + self.fileserver_handler = fileserver_handler + else: + self.fileserver_handler = lambda self, sock, addr: None + + self.sam_ip, self.sam_port = config.i2p_sam.split(":") + self.sam_port = int(self.sam_port) + + # Test SAM port + if config.i2p != "disable": + try: + assert self.connect(), "No connection" + self.log.debug("I2P SAM port %s check ok" % config.i2p_sam) + except Exception as err: + self.log.debug("I2P SAM port %s check error: %s" % (config.i2p_sam, err)) + self.enabled = False + + def connect(self): + if not self.enabled: + return False + self.site_dests = {} + self.dest_conns = {} + self.dest_servs = {} + + self.log.debug("Connecting to %s:%s" % (self.sam_ip, self.sam_port)) + with self.lock: + try: + socket.checkAPIConnection((self.sam_ip, self.sam_port)) + self.status = "Connected" + return True + except Exception as err: + self.status = "Error (%s)" % err + self.log.error("I2P SAM connect error: %s" % Debug.formatException(err)) + self.enabled = False + return False + + def disconnect(self): + for server in self.dest_servs: + server.stop() + self.dest_conns = {} + self.dest_servs = {} + + def startDests(self): + if self.enabled: + self.log.debug("Start Destinations") + self.start_dests = True + + def addDest(self, site_address=None): + sock = socket.socket(socket.AF_I2P, socket.SOCK_STREAM, + samaddr=(self.sam_ip, self.sam_port)) + try: + sock.setblocking(0) + sock.bind(None, site_address) # Transient Destination, tied to site address + sock.listen() + server = StreamServer( + sock, self.fileserver_handler, spawn=Pool(1000) + ) + server.start() + dest = sock.getsockname() + self.dest_conns[dest] = sock + self.dest_servs[dest] = server + self.status = "OK (%s Destinations running)" % len(self.dest_conns) + SiteManager.peer_blacklist.append((dest.base64()+".i2p", 0)) + return dest + except Exception as err: + self.status = "SESSION CREATE error (%s)" % err + self.log.error("I2P SESSION CREATE error: %s" % Debug.formatException(err)) + return False + + def delDest(self, dest): + if dest in self.dest_servs: + self.dest_servs[dest].stop() + del self.dest_conns[dest] + del self.dest_servs[dest] + self.status = "OK (%s Destinations running)" % len(self.dest_conns) + return True + else: + self.status = "Tried to delete non-existent Destination" + self.log.error("I2P error: Tried to delete non-existent") + self.disconnect() + return False + + def getDest(self, site_address): + with self.lock: + if not self.enabled: + return None + if self.start_dests: # Different Destination for every site + dest = self.site_dests.get(site_address) + else: # Same Destination for every site + dest = self.site_dests.get("global") + site_address = "global" + if not dest: + self.site_dests[site_address] = self.addDest(site_address) + dest = self.site_dests[site_address] + self.log.debug("Created new Destination for %s: %s" % (site_address, dest)) + return dest + + def getPrivateDest(self, addr): + dest = addr if isinstance(addr, Destination) else getDest(addr) + return self.dest_conns[dest].getPrivateDest() + + def createSocket(self, site_address, dest, port): + if not self.enabled: + return False + if dest.endswith(".i2p") and not dest.endswith(".b32.i2p"): + dest = Destination(raw=dest[:-4], b64=True) + self.log.debug("Creating new socket to %s:%s" % + (dest.base32() if isinstance(dest, Destination) else dest, port)) + sock = socket.socket(socket.AF_I2P, socket.SOCK_STREAM, + samaddr=(self.sam_ip, self.sam_port)) + sock.connect((dest, int(port)), site_address) + return sock + + def lookup(self, name): + return socket.lookup(name, (self.sam_ip, self.sam_port)) + + def urlopen(self, site_address, url, timeout): + handler = I2PHTTPHandler(self, site_address) + opener = urllib.request.build_opener(handler) + return opener.open(url, timeout=50) diff --git a/src/I2P/__init__.py b/src/I2P/__init__.py new file mode 100644 index 000000000..e7b7dee27 --- /dev/null +++ b/src/I2P/__init__.py @@ -0,0 +1 @@ +from .I2PManager import I2PManager diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 03cc1f47f..c89a7d007 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -126,6 +126,8 @@ def __repr__(self): def packMyAddress(self): if self.ip.endswith(".onion"): return helper.packOnionAddress(self.ip, self.port) + if self.ip.endswith(".i2p"): + return helper.packI2PAddress(self.ip, self.port) else: return helper.packAddress(self.ip, self.port) @@ -273,6 +275,8 @@ def pex(self, site=None, need_num=5): request = {"site": site.address, "peers": packed_peers["ipv4"], "need": need_num} if packed_peers["onion"]: request["peers_onion"] = packed_peers["onion"] + if packed_peers["i2p"]: + request["peers_i2p"] = packed_peers["i2p"] if packed_peers["ipv6"]: request["peers_ipv6"] = packed_peers["ipv6"] res = self.request("pex", request) @@ -299,6 +303,12 @@ def pex(self, site=None, need_num=5): if site.addPeer(*address, source="pex"): added += 1 + # Add I2P + for peer in res.get("peers_i2p", []): + address = helper.unpackI2PAddress(peer) + if site.addPeer(*address): + added += 1 + if added: self.log("Added peers using pex: %s" % added) @@ -331,7 +341,7 @@ def findHashIds(self, hash_ids): back = collections.defaultdict(list) - for ip_type in ["ipv4", "ipv6", "onion"]: + for ip_type in ["ipv4", "ipv6", "onion", "i2p"]: if ip_type == "ipv4": key = "peers" else: @@ -339,6 +349,8 @@ def findHashIds(self, hash_ids): for hash, peers in list(res.get(key, {}).items())[0:30]: if ip_type == "onion": unpacker_func = helper.unpackOnionAddress + elif ip_type == "i2p": + unpacker_func = helper.unpackI2PAddress else: unpacker_func = helper.unpackAddress diff --git a/src/Site/Site.py b/src/Site/Site.py index d61793072..1f169dedf 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -897,6 +897,8 @@ def getConnectablePeers(self, need_num=5, ignore=[], allow_private=True): continue # No connection if peer.ip.endswith(".onion") and not self.connection_server.tor_manager.enabled: continue # Onion not supported + if peer.ip.endswith(".i2p") and not self.connection_server.i2p_manager.enabled: + continue # I2P not supported if peer.key in ignore: continue # The requester has this peer if time.time() - peer.connection.last_recv_time > 60 * 60 * 2: # Last message more than 2 hours ago @@ -937,6 +939,8 @@ def getRecentPeers(self, need_num): need_more = need_num - len(found) if not self.connection_server.tor_manager.enabled: peers = [peer for peer in self.peers.values() if not peer.ip.endswith(".onion")] + elif not self.connection_server.i2p_manager.enabled: + peers = [peer for peer in self.peers.values() if not peer.ip.endswith(".i2p")] else: peers = list(self.peers.values()) @@ -969,6 +973,21 @@ def getConnectedPeers(self): if not peer.connection: peer.connect(connection) back.append(peer) + + i2p_manager = self.connection_server.i2p_manager + for connection in self.connection_server.connections: + if not connection.connected and time.time() - connection.start_time > 20: # Still not connected after 20s + continue + peer = self.peers.get("%s:%s" % (connection.ip, connection.port)) + if peer: + if connection.ip.endswith(".i2p") and connection.target_dest and i2p_manager.start_dests: + # Check if the connection is made with the i2p address created for the site + valid_target_i2p = (i2p_manager.getDest(self.address), i2p_manager.getDest("global").base64()) + if connection.target_dest not in valid_target_onions: + continue + if not peer.connection: + peer.connect(connection) + back.append(peer) return back # Cleanup probably dead peers and close connection if too much diff --git a/src/Site/SiteAnnouncer.py b/src/Site/SiteAnnouncer.py index 2fd63e824..0445795c9 100644 --- a/src/Site/SiteAnnouncer.py +++ b/src/Site/SiteAnnouncer.py @@ -39,6 +39,9 @@ def getSupportedTrackers(self): if not self.site.connection_server.tor_manager.enabled: trackers = [tracker for tracker in trackers if ".onion" not in tracker] + if not self.site.connection_server.i2p_manager.enabled: + trackers = [tracker for tracker in trackers if ".i2p" not in tracker] + trackers = [tracker for tracker in trackers if self.getAddressParts(tracker)] # Remove trackers with unknown address if "ipv6" not in self.site.connection_server.supported_ip_types: @@ -67,6 +70,8 @@ def getOpenedServiceTypes(self): back.append(ip_type) if self.site.connection_server.tor_manager.start_onions: back.append("onion") + if self.site.connection_server.i2p_manager.start_dests: + back.append("i2p") return back @util.Noparallel(blocking=False) diff --git a/src/Test/TestI2P.py b/src/Test/TestI2P.py new file mode 100644 index 000000000..a6a91b74a --- /dev/null +++ b/src/Test/TestI2P.py @@ -0,0 +1,135 @@ +import pytest +import time + +from File import FileServer + +# stats.i2p +TEST_B64 = 'Okd5sN9hFWx-sr0HH8EFaxkeIMi6PC5eGTcjM1KB7uQ0ffCUJ2nVKzcsKZFHQc7pLONjOs2LmG5H-2SheVH504EfLZnoB7vxoamhOMENnDABkIRGGoRisc5AcJXQ759LraLRdiGSR0WTHQ0O1TU0hAz7vAv3SOaDp9OwNDr9u902qFzzTKjUTG5vMTayjTkLo2kOwi6NVchDeEj9M7mjj5ySgySbD48QpzBgcqw1R27oIoHQmjgbtbmV2sBL-2Tpyh3lRe1Vip0-K0Sf4D-Zv78MzSh8ibdxNcZACmZiVODpgMj2ejWJHxAEz41RsfBpazPV0d38Mfg4wzaS95R5hBBo6SdAM4h5vcZ5ESRiheLxJbW0vBpLRd4mNvtKOrcEtyCvtvsP3FpA-6IKVswyZpHgr3wn6ndDHiVCiLAQZws4MsIUE1nkfxKpKtAnFZtPrrB8eh7QO9CkH2JBhj7bG0ED6mV5~X5iqi52UpsZ8gnjZTgyG5pOF8RcFrk86kHxAAAA' + +@pytest.mark.usefixtures("resetSettings") +@pytest.mark.usefixtures("resetTempSettings") +class TestI2P: + def testAddDest(self, i2p_manager): + # Add + dest = i2p_manager.addDest() + assert dest + assert dest in i2p_manager.dest_conns + + # Delete + assert i2p_manager.delDest(dest) + assert dest not in i2p_manager.dest_conns + + def testSignDest(self, i2p_manager): + dest = i2p_manager.addDest() + + # Sign + sign = i2p_manager.getPrivateDest(dest).sign("hello") + assert len(sign) == dest.signature_size() + + # Verify + assert dest.verify("hello", sign) + assert not dest.verify("not hello", sign) + + # Delete + i2p_manager.delDest(dest) + + @pytest.mark.skipif(not pytest.config.getvalue("slow"), reason="--slow not requested (takes around ~ 1min)") + def testConnection(self, i2p_manager, file_server, site, site_temp): + file_server.i2p_manager.start_dests = True + dest = file_server.i2p_manager.getDest(site.address) + assert dest + print("Connecting to", dest.base32()) + for retry in range(5): # Wait for Destination creation + time.sleep(10) + try: + connection = file_server.getConnection(dest.base64()+".i2p", 1544) + if connection: + break + except Exception as err: + continue + assert connection.handshake + assert not connection.handshake["peer_id"] # No peer_id for I2P connections + + # Return the same connection without site specified + assert file_server.getConnection(dest.base64()+".i2p", 1544) == connection + # No reuse for different site + assert file_server.getConnection(dest.base64()+".i2p", 1544, site=site) != connection + assert file_server.getConnection(dest.base64()+".i2p", 1544, site=site) == file_server.getConnection(dest.base64()+".i2p", 1544, site=site) + site_temp.address = "1OTHERSITE" + assert file_server.getConnection(dest.base64()+".i2p", 1544, site=site) != file_server.getConnection(dest.base64()+".i2p", 1544, site=site_temp) + + # Only allow to query from the locked site + file_server.sites[site.address] = site + connection_locked = file_server.getConnection(dest.base64()+".i2p", 1544, site=site) + assert "body" in connection_locked.request("getFile", {"site": site.address, "inner_path": "content.json", "location": 0}) + assert connection_locked.request("getFile", {"site": "1OTHERSITE", "inner_path": "content.json", "location": 0})["error"] == "Invalid site" + + def testPex(self, file_server, site, site_temp): + # Register site to currently running fileserver + site.connection_server = file_server + file_server.sites[site.address] = site + # Create a new file server to emulate new peer connecting to our peer + file_server_temp = FileServer("127.0.0.1", 1545) + site_temp.connection_server = file_server_temp + file_server_temp.sites[site_temp.address] = site_temp + # We will request peers from this + peer_source = site_temp.addPeer("127.0.0.1", 1544) + + # Get ip4 peers from source site + assert peer_source.pex(need_num=10) == 1 # Need >5 to return also return non-connected peers + assert len(site_temp.peers) == 2 # Me, and the other peer + site.addPeer("1.2.3.4", 1555) # Add peer to source site + assert peer_source.pex(need_num=10) == 1 + assert len(site_temp.peers) == 3 + assert "1.2.3.4:1555" in site_temp.peers + + # Get I2P peers from source site + site.addPeer(TEST_B64+".i2p", 1555) + assert TEST_B64+".i2p:1555" not in site_temp.peers + assert peer_source.pex(need_num=10) == 1 # Need >5 to return also return non-connected peers + assert TEST_B64+".i2p:1555" in site_temp.peers + + def testFindHash(self, i2p_manager, file_server, site, site_temp): + file_server.ip_incoming = {} # Reset flood protection + file_server.sites[site.address] = site + assert file_server.i2p_manager == None + file_server.i2p_manager = i2p_manager + + client = FileServer("127.0.0.1", 1545) + client.sites[site_temp.address] = site_temp + site_temp.connection_server = client + + # Add file_server as peer to client + peer_file_server = site_temp.addPeer("127.0.0.1", 1544) + + assert peer_file_server.findHashIds([1234]) == {} + + # Add fake peer with requred hash + fake_peer_1 = site.addPeer(TEST_B64+".i2p", 1544) + fake_peer_1.hashfield.append(1234) + fake_peer_2 = site.addPeer("1.2.3.5", 1545) + fake_peer_2.hashfield.append(1234) + fake_peer_2.hashfield.append(1235) + fake_peer_3 = site.addPeer("1.2.3.6", 1546) + fake_peer_3.hashfield.append(1235) + fake_peer_3.hashfield.append(1236) + + assert peer_file_server.findHashIds([1234, 1235]) == { + 1234: [('1.2.3.5', 1545), (TEST_B64+".i2p", 1544)], + 1235: [('1.2.3.6', 1546), ('1.2.3.5', 1545)] + } + + # Test my address adding + site.content_manager.hashfield.append(1234) + my_i2p_address = i2p_manager.getDest(site_temp.address).base64()+".i2p" + + res = peer_file_server.findHashIds([1234, 1235]) + assert res[1234] == [('1.2.3.5', 1545), (TEST_B64+".i2p", 1544), (my_i2p_address, 1544)] + assert res[1235] == [('1.2.3.6', 1546), ('1.2.3.5', 1545)] + + # Reset + file_server.i2p_manager = None + + def testSiteDest(self, i2p_manager): + assert i2p_manager.getDest("address1") != i2p_manager.getDest("address2") + assert i2p_manager.getDest("address1") == i2p_manager.getDest("address1") diff --git a/src/Test/conftest.py b/src/Test/conftest.py index c8739086f..a64edbdac 100644 --- a/src/Test/conftest.py +++ b/src/Test/conftest.py @@ -74,6 +74,7 @@ def pytest_collection_modifyitems(config, items): config.debug_socket = True # Use test data for unittests config.verbose = True # Use test data for unittests config.tor = "disable" # Don't start Tor client +config.i2p = "disable" # Don't start I2P client config.trackers = [] config.data_dir = TEST_DATA_PATH # Use test data for unittests if "ZERONET_LOG_DIR" in os.environ: @@ -126,6 +127,7 @@ def filter(self, record): from Crypt import CryptBitcoin from Ui import UiWebsocket from Tor import TorManager +from I2P import I2PManager from Content import ContentDb from util import RateLimit from Db import Db @@ -495,3 +497,13 @@ def disableLog(): yield None # Wait until all test done logging.getLogger('').setLevel(logging.getLevelName(logging.CRITICAL)) +@pytest.fixture(scope="session") +def i2p_manager(): + try: + i2p_manager = I2PManager() + i2p_manager.enabled = True + assert i2p_manager.connect(), "No connection" + i2p_manager.startDests() + except Exception as err: + raise pytest.skip("Test requires I2P with SAM port: %s, %s" % (config.i2p_sam, err)) + return i2p_manager diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py index 2f982e1d3..fa0350197 100644 --- a/src/Ui/UiWebsocket.py +++ b/src/Ui/UiWebsocket.py @@ -47,7 +47,7 @@ def start(self): self.site.page_requested = True # Dont add connection notification anymore import main file_server = main.file_server - if not file_server.port_opened or file_server.tor_manager.start_onions is None: + if not file_server.port_opened or (file_server.tor_manager.start_onions or file_server.i2p_manager.start_dests) is None: self.site.page_requested = False # Not ready yet, check next time else: try: @@ -302,6 +302,8 @@ def formatServerInfo(self): "tor_status": file_server.tor_manager.status, "tor_has_meek_bridges": file_server.tor_manager.has_meek_bridges, "tor_use_bridges": config.tor_use_bridges, + "i2p_enabled": file_server.i2p_manager.enabled, + "i2p_status": file_server.i2p_manager.status, "ui_ip": config.ui_ip, "ui_port": config.ui_port, "version": config.version, @@ -537,7 +539,7 @@ def cbSitePublish(self, to, site, thread, notification=True, callback=True): else: if len(site.peers) == 0: import main - if any(main.file_server.port_opened.values()) or main.file_server.tor_manager.start_onions: + if any(main.file_server.port_opened.values()) or main.file_server.tor_manager.start_onions or main.file_server.i2p_manager.start_onions: if notification: self.cmd("notification", ["info", _["No peers found, but your content is ready to access."]]) if callback: diff --git a/src/util/helper.py b/src/util/helper.py index 61455b085..48b18e1d3 100644 --- a/src/util/helper.py +++ b/src/util/helper.py @@ -115,7 +115,7 @@ def shellquote(*args): def packPeers(peers): - packed_peers = {"ipv4": [], "ipv6": [], "onion": []} + packed_peers = {"ipv4": [], "ipv6": [], "onion": [], "i2p": []} for peer in peers: try: ip_type = getIpType(peer.ip) @@ -154,6 +154,17 @@ def packOnionAddress(onion, port): def unpackOnionAddress(packed): return base64.b32encode(packed[0:-2]).lower().decode() + ".onion", struct.unpack("H", packed[-2:])[0] +# Destination, port to packed (389+)-byte format +def packI2PAddress(dest, port): + if not isinstance(dest, Destination): + dest = dest.replace(".i2p", "") + dest = Destination(raw=dest, b64=True) + return dest.serialize() + struct.pack("H", port) + + +# From (389+)-byte format to Destination, port +def unpackI2PAddress(packed): + return Destination(raw=packed[0:-2]).base64() + ".i2p", struct.unpack("H", packed[-2:])[0] # Get dir from file # Return: data/site/content.json -> data/site/ From 370c2b04d1b5cac5e3b0fc6af0b1f1dadca353a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20K=C3=BCthe?= Date: Sun, 22 Oct 2023 19:37:05 +0200 Subject: [PATCH 2/2] Fix error when closing sockets --- src/Ui/UiServer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Ui/UiServer.py b/src/Ui/UiServer.py index 61943ada1..b38d9d03e 100644 --- a/src/Ui/UiServer.py +++ b/src/Ui/UiServer.py @@ -192,7 +192,9 @@ def stop(self): from Debug import DebugReloader DebugReloader.watcher.stop() - self.server.socket.close() + for socket in self.server.sockets: + socket.stop() + self.server.stop() self.running = False time.sleep(1)