Fixed migrations, bubble updates, add Jenkins progress

app/loader/interfaces/jenkins_loader.py

@@ -1,7 +1,20 @@
-import urllib2, json, re
+import urllib2, json, re, sys, math
 
 class loader_jenkins:
 
+	def makeProgressBar(self, message):
+		toolbar_width = 20
+
+		sys.stdout.write("\n" + message + "\n")
+
+		sys.stdout.write("[%s]" % (" " * toolbar_width))
+		sys.stdout.flush()
+		sys.stdout.write("\b" * (toolbar_width+1))
+
+	def incrementProgressBar(self):
+		sys.stdout.write("-")
+		sys.stdout.flush()
+
 	def apiURL(self, url):
 		if not url.endswith('/'): url += '/'
 		return url + 'api/json?pretty=true'
@@ -49,7 +62,14 @@ def load(self, channel, last_build):
 		data = self.getJSON(channel['url'])
 		builds = []
 
-		for build in data['builds']:
+		every = math.floor(len(data['builds']) / 20)
+		if every > 0:
+			self.makeProgressBar('Grabbing from Jenkins...')
+
+		for i, build in enumerate(data['builds']):
+			if every > 0 and i % every == 0:
+				self.incrementProgressBar()
+
 			if build['number'] <= last_build: continue
 
 			out = self.getBuildData(build)

app/loader/loader.py

@@ -1,4 +1,4 @@
-import os, yggdrasil
+import os, yggdrasil, sys
 from interfaces import *
 from gdn import app
 from gdn.models import Version, Build
@@ -57,6 +57,8 @@ def load():
 			channel = adder.addChannel(channel_data, jar_obj)
 			l = getLoader(channel_data['interface'])
 
+			sys.stdout.write("\n\nLoading builds for %s" % (source['name'] + '#' + channel_data['name']))
+
 			if not l: continue;
 
 			data = getLastBuild(channel)
@@ -67,4 +69,5 @@ def load():
 			for build in l.load(channel_data, last_build):
 				adder.addBuild(build, channel)
 
-	adder.commit()
+	adder.commit()
+	sys.stdout.write("\n\n")

app/loader/yggdrasil.py

@@ -2,6 +2,7 @@
 from gdn.models import Jar, Channel, Version, Build
 
 from datetime import datetime
+import sys
 
 class Yggdrasil():
 	jars = {}
@@ -73,12 +74,12 @@ def addBuild(self, data, channel):
 		data['version_id'] = version
 
 		self.getOrMake(model = Build, data = data, where = { 'build': data['build'], 'version_id': data['version_id'] })
-		self.bubbleUpdate(channel);
+		self.bubbleUpdate(version);
 
-	def bubbleUpdate(self, channel):
-		channel_model = self.channels[channel]
-		version_model = self.versions[channel_model.version_id]
-		jar_model = self.jars[version_model.jar_id]
+	def bubbleUpdate(self, version):
+		version_model = self.versions[version]
+		channel_model = self.channels[version_model.channel_id]
+		jar_model = self.jars[channel_model.jar_id]
 
 		for parent in [channel_model, version_model, jar_model]:
 			parent.updated_at = datetime.now()

app/migrations/versions/3e7f3df84c9e_.py → app/migrations/versions/3df41fbd302d_.py

@@ -1,13 +1,13 @@
 """empty message
 
-Revision ID: 3e7f3df84c9e
+Revision ID: 3df41fbd302d
 Revises: None
-Create Date: 2013-12-10 16:23:31.482098
+Create Date: 2013-12-28 10:47:21.461151
 
 """
 
 # revision identifiers, used by Alembic.
-revision = '3e7f3df84c9e'
+revision = '3df41fbd302d'
 down_revision = None
 
 from alembic import op
@@ -18,50 +18,54 @@ def upgrade():
     ### commands auto generated by Alembic - please adjust! ###
     op.create_table('jars',
     sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('name', sa.String(length=32), nullable=True),
-    sa.Column('site_url', sa.String(length=100), nullable=True),
-    sa.Column('updated_at', sa.TIMESTAMP(), server_default=sa.text('current_timestamp()')),
+    sa.Column('name', sa.String(length=32), nullable=False),
+    sa.Column('site_url', sa.String(length=100), nullable=False),
+    sa.Column('created_at', sa.TIMESTAMP(), nullable=False),
+    sa.Column('updated_at', sa.TIMESTAMP(), nullable=False),
     sa.PrimaryKeyConstraint('id'),
     sa.UniqueConstraint('name')
     )
+    op.create_table('requests',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('requests', sa.Integer(), nullable=True),
+    sa.Column('ip', sa.String(length=45), nullable=True),
+    sa.Column('updated_at', sa.TIMESTAMP(), nullable=True),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('ip', name='uix_1')
+    )
     op.create_table('channels',
     sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('jar_id', sa.Integer()),
-    sa.Column('name', sa.String(length=32)),
-    sa.Column('updated_at', sa.TIMESTAMP(), server_default=sa.text('current_timestamp()')),
+    sa.Column('jar_id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(length=32), nullable=False),
+    sa.Column('created_at', sa.TIMESTAMP(), nullable=False),
+    sa.Column('updated_at', sa.TIMESTAMP(), nullable=False),
     sa.ForeignKeyConstraint(['jar_id'], ['jars.id'], ),
     sa.PrimaryKeyConstraint('id'),
     sa.UniqueConstraint('jar_id','name', name='uix_1')
     )
     op.create_table('versions',
     sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('channel_id', sa.Integer()),
-    sa.Column('version', sa.String(length=32)),
-    sa.Column('updated_at', sa.TIMESTAMP(), server_default=sa.text('current_timestamp()')),
+    sa.Column('channel_id', sa.Integer(), nullable=False),
+    sa.Column('version', sa.String(length=32), nullable=False),
+    sa.Column('created_at', sa.TIMESTAMP(), nullable=False),
+    sa.Column('updated_at', sa.TIMESTAMP(), nullable=False),
     sa.ForeignKeyConstraint(['channel_id'], ['channels.id'], ),
     sa.PrimaryKeyConstraint('id'),
     sa.UniqueConstraint('channel_id','version', name='uix_1')
     )
     op.create_table('builds',
     sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('version_id', sa.Integer()),
-    sa.Column('build', sa.Integer()),
+    sa.Column('version_id', sa.Integer(), nullable=False),
+    sa.Column('build', sa.Integer(), nullable=False),
     sa.Column('size', sa.Integer(), nullable=True),
     sa.Column('checksum', sa.String(length=32), nullable=True),
-    sa.Column('url', sa.String(length=150)),
-    sa.Column('created_at', sa.TIMESTAMP(), server_default=sa.text('current_timestamp()')),
+    sa.Column('url', sa.String(length=150), nullable=False),
+    sa.Column('created_at', sa.TIMESTAMP(), nullable=False),
+    sa.Column('updated_at', sa.TIMESTAMP(), nullable=False),
     sa.ForeignKeyConstraint(['version_id'], ['versions.id'], ),
     sa.PrimaryKeyConstraint('id'),
     sa.UniqueConstraint('version_id','build', name='uix_1')
     )
-    op.create_table('requests',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('requests', sa.Integer()),
-    sa.Column('ip', sa.String(length=45)),
-    sa.Column('updated_at', sa.TIMESTAMP(), server_default=sa.text('current_timestamp()'), nullable=True),
-    sa.PrimaryKeyConstraint('id'),
-    sa.UniqueConstraint('ip', name='uix_1')
-    )
     ### end Alembic commands ###
 
 
@@ -70,5 +74,6 @@ def downgrade():
     op.drop_table('builds')
     op.drop_table('versions')
     op.drop_table('channels')
+    op.drop_table('requests')
     op.drop_table('jars')
     ### end Alembic commands ###