Merge pull request #4473 from xoriole/trust-graph

Trust graph

Tribler/Core/APIImplementation/LaunchManyCore.py

@@ -77,7 +77,7 @@ def __init__(self):
 
         self.shutdownstarttime = None
 
-        self.bootstrap_download = None
+        self.bootstrap = None
 
         # modules
         self.api_manager = None
@@ -344,22 +344,9 @@ def init(self):
 
         self.session.set_download_states_callback(self.sesscb_states_callback)
 
-        payout_enabled = False
         if self.session.config.get_ipv8_enabled() and self.session.config.get_trustchain_enabled():
             self.payout_manager = PayoutManager(self.trustchain_community, self.dht_community)
-            payout_enabled = True
 
-        if self.session.config.get_bootstrap_enabled():
-            if not payout_enabled:
-                self._logger.warn("Running bootstrap without payout enabled")
-            bootstrap = Bootstrap(self.session.config.get_state_dir())
-            bootstrap_file = os.path.join(self.session.config.get_state_dir(), 'bootstrap.block')
-            if os.path.exists(bootstrap_file):
-                self.bootstrap_download = bootstrap.start_initial_seeder(self.session.start_download_from_tdef,
-                                                                         bootstrap_file)
-            else:
-                self.bootstrap_download = bootstrap.start_by_infohash(self.session.start_download_from_tdef,
-                                                                      self.session.config.get_bootstrap_infohash())
         self.initComplete = True
 
     def add(self, tdef, dscfg, pstate=None, setupDelay=0, hidden=False,
@@ -567,6 +554,8 @@ def sesscb_states_callback(self, states_list):
                 for peer in download.get_peerlist():
                     if peer["extended_version"].startswith('Tribler'):
                         self.payout_manager.update_peer(unhexlify(peer["id"]), infohash, peer["dtotal"])
+                        if self.bootstrap and hexlify(infohash) == self.session.config.get_bootstrap_infohash():
+                            self.bootstrap.fetch_bootstrap_peers()
 
         self.previous_active_downloads = new_active_downloads
         if do_checkpoint:
@@ -692,6 +681,18 @@ def do_remove():
 
         reactor.callFromThread(do_remove)
 
+    def start_bootstrap_download(self):
+        if self.session.config.get_bootstrap_enabled():
+            if not self.payout_manager:
+                self._logger.warn("Running bootstrap without payout enabled")
+            self.bootstrap = Bootstrap(self.session.config.get_state_dir(), dht=self.dht_community)
+            if os.path.exists(self.bootstrap.bootstrap_file):
+                self.bootstrap.start_initial_seeder(self.session.start_download_from_tdef,
+                                                    self.bootstrap.bootstrap_file)
+            else:
+                self.bootstrap.start_by_infohash(self.session.start_download_from_tdef,
+                                                 self.session.config.get_bootstrap_infohash())
+
     @inlineCallbacks
     def early_shutdown(self):
         """ Called as soon as Session shutdown is initiated. Used to start

Tribler/Core/Modules/TrustCalculation/__init__.py

@@ -0,0 +1,4 @@
+"""
+The TrustCalculation package contains the procedures required for
+the monitoring of transactions and the calculation of trust
+"""

Tribler/Core/Modules/TrustCalculation/graph_positioning.py

@@ -0,0 +1,80 @@
+from __future__ import absolute_import, division
+
+import random
+
+import networkx as nx
+
+
+class GraphPositioning(object):
+    """
+    This class is for the calculation of the positions of the nodes of
+    a given tree and from the perspective of a given central node
+    """
+
+    @staticmethod
+    def hierarchy_pos(G, root=None, width=1., vert_gap=0.2,
+                      vert_loc=0, xcenter=0.5):
+        """
+        Taken from: https://bit.ly/2tetWxf
+
+        If the graph is a tree this will return the positions to plot this in a
+        hierarchical layout.
+
+        G: the graph (must be a tree)
+
+        root: the root node of current branch
+        - if the tree is directed and this is not given,
+          the root will be found and used
+        - if the tree is directed and this is given, then the positions
+          will be just for the descendants of this node.
+        - if the tree is undirected and not given, then a random
+          choice will be used.
+
+        width: horizontal space allocated for this branch - avoids overlap
+          with other branches
+
+        vert_gap: gap between levels of hierarchy
+
+        vert_loc: vertical location of root
+
+        xcenter: horizontal location of root
+        """
+
+        if not nx.is_tree(G):
+            raise TypeError('cannot use hierarchy_pos on a'
+                            ' graph that is not a tree')
+
+        if root is None:
+            if isinstance(G, nx.DiGraph):
+                # allows back compatibility with nx version 1.11
+                root = next(iter(nx.topological_sort(G)))
+            else:
+                root = random.choice(list(G.nodes))
+
+        def _hierarchy_pos(G, root, width=1., vert_gap=0.2, vert_loc=0,
+                           xcenter=0.5, pos=None, parent=None):
+            """
+            see hierarchy_pos docstring for most arguments
+            pos: a dict saying where all nodes go if they have been assigned
+            parent: parent of this branch. - only affects it if non-directed
+
+            """
+
+            if pos is None:
+                pos = {root: (xcenter, vert_loc)}
+            else:
+                pos[root] = (xcenter, vert_loc)
+            children = list(G.neighbors(root))
+            if not isinstance(G, nx.DiGraph) and parent is not None:
+                children.remove(parent)
+            if children:
+                dx = width/len(children)
+                nextx = xcenter - width/2 - dx/2
+                for child in children:
+                    nextx += dx
+                    pos = _hierarchy_pos(G, child, width=dx, vert_gap=vert_gap,
+                                         vert_loc=vert_loc-vert_gap,
+                                         xcenter=nextx, pos=pos, parent=root)
+            return pos
+
+        return _hierarchy_pos(G, root, width, vert_gap, vert_loc, xcenter)

Tribler/Core/Modules/TrustCalculation/local_view.py

@@ -0,0 +1,128 @@
+from __future__ import absolute_import, division
+
+import math
+
+import networkx as nx
+
+from Tribler.Core.Modules.TrustCalculation.graph_positioning import GraphPositioning as gpos
+
+
+class NodeVision(object):
+    """
+    This object is used for laying out the nodes of a graph according to \
+    the local vision of a specific node (root node)
+    """
+
+    def __init__(self, root_node):
+        """
+        :param graph: Peer network
+        :param root_node: Transaction keypair of the node
+        """
+
+        self.graph = nx.DiGraph()
+        self.graph.add_node(root_node)
+
+        self.root_node = root_node
+        self.bfs_tree = {}  # Bfs tree rooted at the peer
+        self.component = None  # The connected component which includes the peer
+
+        self.pos = self.lay_down_nodes()
+        self.component_pos = dict(self.pos)
+
+        self.update_component()
+
+    def set_root_node(self, rootnode):
+        self.root_node = rootnode
+        self.pos = self.lay_down_nodes()
+
+    @property
+    def n_nodes(self):
+        return self.graph.number_of_nodes()
+
+    @property
+    def node_positions(self):
+        return dict(self.graph.nodes(data='pos'))
+
+    def reposition_nodes(self):
+        self.pos = self.lay_down_nodes()
+        self.component_pos = self.normalize_positions_dict()
+
+    def lay_down_nodes(self):
+        """
+        Given a directed graph, finds the connected component which includes
+        the root node and then determines positions in the circular view.
+
+        :return: Positions of the nodes from the perspective of the root node
+        !!! returned dict does not contain positions of unconnected nodes!
+        """
+        # Find undirected graph (needed for connected component discovery)
+        gr_undirected = self.graph.to_undirected()
+
+        # Remove disconnected nodes from the graph
+        component_nodes = nx.node_connected_component(gr_undirected, self.root_node)
+        for node in list(gr_undirected.nodes()):
+            if node not in component_nodes:
+                gr_undirected.remove_node(node)
+
+        # Find bfs tree of the connected components
+        bfs_tree = nx.bfs_tree(gr_undirected, self.root_node)
+        self.bfs_tree[self.root_node] = bfs_tree
+
+        # Position the nodes in a circular fashion according to the bfs tree
+        pos = gpos.hierarchy_pos(bfs_tree, self.root_node,
+                                 width=2 * math.pi, xcenter=0.5)
+        new_pos = {u: (r * math.cos(theta), r * math.sin(theta))
+                   for u, (theta, r) in pos.items()}
+
+        # Set positions to the networkx object
+        nx.set_node_attributes(self.graph, new_pos, 'pos')
+
+        # Also, return the positions
+        return new_pos
+
+    def normalize_positions_dict(self, width=0.80, margin=0.05):
+        poslist = [v for v in self.pos.values() if v is not None]
+        minx = min(poslist, key=lambda t: t[0])[0]
+        miny = min(poslist, key=lambda t: t[1])[1]
+        maxx = max(poslist, key=lambda t: t[0])[0]
+        maxy = max(poslist, key=lambda t: t[1])[1]
+        xinterval = (maxx - minx)
+        yinterval = (maxy - miny)
+
+        # To escape from division by zero:
+        if xinterval == 0:
+            xinterval = 1.0
+        if yinterval == 0:
+            yinterval = 1.0
+
+        newposlist = {}
+
+        for node, pos in self.pos.items():
+            if pos is not None:
+                nposx = ((pos[0] - minx) / xinterval) * width + margin
+                nposy = ((pos[1] - miny) / yinterval) * width + margin
+                newposlist[node] = (nposx, nposy)
+
+        return newposlist
+
+    def add_transactions(self, transactions):
+        for tr in transactions:
+            self.add_edge_to_graph(tr['downloader'],
+                                   tr['uploader'],
+                                   tr['amount'])
+
+    def add_edge_to_graph(self, n1, n2, w):
+        if n1 in self.graph and n2 in self.graph.successors(n1):
+            self.graph[n1][n2]['weight'] *= 0.8
+            self.graph[n1][n2]['weight'] += (0.2 * w)
+        else:
+            self.graph.add_edge(n1, n2, weight=w)
+
+    def update_component(self):
+        H = self.graph.to_undirected()
+        self.component = nx.DiGraph(self.graph)
+
+        component_nodes = nx.node_connected_component(H, self.root_node)
+        for node in self.graph:
+            if node not in component_nodes:
+                self.component.remove_node(node)

Tribler/Core/Modules/restapi/root_endpoint.py

@@ -17,6 +17,7 @@
 from Tribler.Core.Modules.restapi.statistics_endpoint import StatisticsEndpoint
 from Tribler.Core.Modules.restapi.torrentinfo_endpoint import TorrentInfoEndpoint
 from Tribler.Core.Modules.restapi.trustchain_endpoint import TrustchainEndpoint
+from Tribler.Core.Modules.restapi.trustview_endpoint import TrustViewEndpoint
 from Tribler.Core.Modules.restapi.wallets_endpoint import WalletsEndpoint
 from Tribler.pyipv8.ipv8.REST.root_endpoint import RootEndpoint as IPV8RootEndpoint
 
@@ -53,6 +54,7 @@ def start_endpoints(self):
             b"debug": DebugEndpoint,
             b"shutdown": ShutdownEndpoint,
             b"trustchain": TrustchainEndpoint,
+            b"trustview": TrustViewEndpoint,
             b"statistics": StatisticsEndpoint,
             b"market": MarketEndpoint,
             b"wallets": WalletsEndpoint,

Tribler/Core/Modules/restapi/trustview_endpoint.py

@@ -0,0 +1,104 @@
+from __future__ import absolute_import
+
+import logging
+from binascii import hexlify, unhexlify
+
+import networkx as nx
+
+from twisted.web import resource
+
+import Tribler.Core.Utilities.json_util as json
+from Tribler.Core.Modules.TrustCalculation.local_view import NodeVision
+from Tribler.Core.simpledefs import DOWNLOAD, UPLOAD
+
+
+class TrustViewEndpoint(resource.Resource):
+
+    def __init__(self, session):
+        resource.Resource.__init__(self)
+        self.session = session
+        self._logger = logging.getLogger(self.__class__.__name__)
+
+        self.node_id = None
+        self.local_view = None
+
+        self.bootstrap = None
+        self.peers = []
+        self.transactions = {}
+        self.initialized = False
+        self.trustchain_db = None
+
+    def initialize_graph(self):
+        if not self.initialized and self.session.lm.trustchain_community and not self.local_view:
+            pub_key = self.session.lm.trustchain_community.my_peer.public_key.key_to_bin()
+            self.node_id = hexlify(pub_key)
+            self.local_view = NodeVision(self.node_id)
+            self.trustchain_db = self.session.lm.trustchain_community.persistence
+            self.initialized = True
+
+            # Start bootstrap download if not already done
+            if not self.session.lm.bootstrap:
+                self.session.lm.start_bootstrap_download()
+
+    @staticmethod
+    def block_to_edge(block):
+        if not block:
+            return None
+
+        diff = block.transaction['up'] - block.transaction['down']
+        if diff < 0:
+            return {'downloader': hexlify(block.public_key),
+                    'uploader': hexlify(block.link_public_key),
+                    'amount': diff * -1
+                   }
+        return {'downloader': hexlify(block.link_public_key),
+                'uploader': hexlify(block.public_key),
+                'amount': diff
+               }
+
+    def load_single_block(self, block):
+        if block.hash not in self.transactions:
+            self.transactions[block.hash] = self.block_to_edge(block)
+
+    def load_blocks(self, blocks):
+        for block in blocks:
+            self.load_single_block(block)
+
+    def render_GET(self, _):
+        self.initialize_graph()
+
+        # Load your 25 latest trustchain blocks
+        pub_key = self.session.lm.trustchain_community.my_peer.public_key.key_to_bin()
+        blocks = self.trustchain_db.get_latest_blocks(pub_key)
+        self.load_blocks(blocks)
+
+        # Load 25 latest blocks of all the users in the database
+        userblocks = self.trustchain_db.get_users()
+        for userblock in userblocks:
+            blocks = self.trustchain_db.get_latest_blocks(unhexlify(userblock['public_key']), limit=25)
+            self.load_blocks(blocks)
+
+        # Add blocks to graph and update your local view
+        self.local_view.add_transactions(self.transactions.values())
+        self.local_view.lay_down_nodes()
+        self.local_view.reposition_nodes()
+        self.local_view.update_component()
+
+        positions = self.local_view.normalize_positions_dict()
+        graph_data = nx.node_link_data(self.local_view.component)
+
+        return json.twisted_dumps({'node_id': self.node_id,
+                                   'graph_data': graph_data,
+                                   'positions': positions,
+                                   'bootstrap': self.get_bootstrap_info(),
+                                   'num_tx': len(self.transactions)
+                                  })
+
+    def get_bootstrap_info(self):
+        if self.session.lm.bootstrap.download and self.session.lm.bootstrap.download.get_state():
+            state = self.session.lm.bootstrap.download.get_state()
+            return {'download': state.get_total_transferred(DOWNLOAD),
+                    'upload': state.get_total_transferred(UPLOAD),
+                    'progress': state.get_progress()
+                   }
+        return {'download': 0, 'upload': 0, 'progress': 0}