merge.py

import re
import collections
import sys

#USE WITH DATA.TXT THAT HAS SITERANK
#run mergeold.py if you need to add siterank
#this tool uses the peerlist.txt generated by grab.py

#Get site ranks
filename = 'peerlist.txt'
textfile = open(filename, 'r')
filetext = textfile.read()
textfile.close()

newsites = {}

for item in filetext.split('\n'):
	if item != "":
		entry = item.split(" ");
		newsites[entry[0]] = entry[1], ' '.join(entry[2:])


#Get existing sites
filename = 'data.txt'
textfile = open(filename, 'r')
data = textfile.read().split('\n')
textfile.close()
#Append rank to existing sites and store to a new data file
f = open("datanew.txt","w")
for line in data:
	lsplit = line.split(":")
	asplit = lsplit[1].strip().split(" ")
	siteaddress = asplit[0]
	newrank = newsites.pop(siteaddress, '-')[0]
	oldrank = asplit[1]
	rank = oldrank

	#Modify this to choose how siterank is updated
	if oldrank == '-':
		rank = newrank
	else:
		if newrank == '-':
			rank = oldrank
		else:
			rank = (int(newrank)+int(oldrank))/2 #newrank 

	f.write(lsplit[0]+":"+siteaddress+" "+str(rank)+"\n")
f.close()

#Store new sites in a separate file for tagging
f = open("newsites.txt","a")
f.write("---\n")
for site in newsites:
	n = newsites.get(site,'-')
	f.write(n[1]+"[:"+site+" "+n[0]+"\n")
f.close()