tumblrdownloader.py

#!/usr/bin/env python

'''
	Diego Martins de Siqueira
	download all images from a Tumblr
'''

from __future__ import unicode_literals

try:
    from urllib.request import urlopen, urlretrieve
    from queue import Queue
except ImportError:
    from urllib2 import urlopen
    from urllib import urlretrieve
    from Queue import Queue

import re
import os
import sys
import argparse
import threading


class DownloadThread(threading.Thread):
    def __init__(self, queue, destfolder, image_prefix):
        super(DownloadThread, self).__init__()

        self.queue 			= queue
        self.destfolder 	= destfolder
        self.image_prefix 	= image_prefix
        self.daemon 		= True

    def run(self):
        while True:
            url = self.queue.get()
            try:
                self.download_url(url)
            except Exception as e:
                print("   Error: %s"%e)
            self.queue.task_done()

    def download_url(self, url):
        image_name = url.split('/')[-1]
        name = self.image_prefix + "_" + image_name
        dest = os.path.join(self.destfolder, name)
        print("[%s] Downloading %s"%(self.ident, image_name))
        urlretrieve(url, dest)

class TumblrDownloader:

	api_url	= 'http://#subdomain#.tumblr.com/api/read?type=photo&num=#chunk#&start=#start#' 

	def __init__(self, subdomain, chunk, output, resolution, tagged, chrono, total, start, threads):
		self._subdomain = subdomain
		self._chunk 	= chunk
		self._output 	= output
		self._resolution = resolution
		self._tagged 	= tagged
		self._chrono 	= chrono
		self._total 	= total
		self._start		= start
		self._threads	= threads

		self.api_url = self.api_url.replace("#subdomain#",self._subdomain)
		self.api_url = self.api_url.replace("#chunk#",str(self._chunk))

		self._image_prefix = self._subdomain;

		self._folder_path = self._output + "/" + self._subdomain + "/" + str(self._resolution)

		if (self._chrono):
			self.api_url += "&chrono=1"

		if self._tagged:
			self.api_url 		+= "&tagged=" + self._tagged
			self._folder_path 	+= "/" + self._tagged
			self._image_prefix	+= "_" + self._tagged;

		self._createfolder()

	def _createfolder(self):
		'''
			if folder does not exist, create it.
		'''

		if not os.path.exists(self._folder_path):
	 		os.makedirs(self._folder_path)

	def download(self):
		'''
			download all images from a Tumblr
		'''

		while True:
			imagelist	=	self._getimages()
			self._start += 	self._chunk

			if not imagelist:
				break

			if self._total:
				imagelist = imagelist[0:self._total]

			self._downloadimage(imagelist)

			if self._total:
				self._total -= len(imagelist);

				if (self._total <= 0):
					break

	def _getimages(self):
		'''
			Get all images returned by Tumblr API
		'''
		site = self.api_url.replace("#start#",str(self._start))

		file = urlopen(site)
		data = file.read().decode('utf8')
		file.close()

		regex		= r"<photo-url max-width=\"" + str(self._resolution) + "\">(.+?)</photo-url>"
		imagelist	= re.findall(regex, data)
		return imagelist

	def _downloadimage(self,url_list):
		'''
			Download a image to subdomain folder.
		'''
		queue = Queue()
		for url in url_list:
			queue.put(url)

		for i in range(self._threads):
			t = DownloadThread(queue, self._folder_path, self._image_prefix)
			t.start()

		queue.join()

def main(argv):
	parser = argparse.ArgumentParser(description="Download all images from a Tumblr")
	parser.add_argument("subdomain", type=str, 
		help="Tumblr subdomain you want to download")
	parser.add_argument("--chunk", type=int, default=20, 
		help="The number of posts to return each call to Tumblrs API. The default is 20, and the maximum is 50.")
	parser.add_argument("--total", type=int,
		help="Total images to download")
	parser.add_argument("--start", type=int, default=0, 
		help="The post offset to start from. The default is 0.")
	parser.add_argument("--output", type=str, default="images", 
		help="Output folder")
	parser.add_argument("--resolution", type=int, default=1280, choices=[1280, 500, 400, 250, 100, 75],
        help="Select Max Width to download. The default is 1280.")
	parser.add_argument("--tagged", type=str,
		help="Download only images with tag")
	parser.add_argument("--chrono", action="store_true", 
		help="Sort in chronological order (oldest first)")
	parser.add_argument("--threads", type=int, default=5, 
		help="Number of parallel downloads. The default is 5.")

	args = parser.parse_args()

	print('Downloading Subdomain: ', args.subdomain)

	try:
		td = TumblrDownloader(args.subdomain,args.chunk,args.output,args.resolution,args.tagged,args.chrono,
			args.total,args.start,args.threads)
		td.download()
		print('All images were downloaded.')
	except KeyboardInterrupt:
		print('Interrupt received, stopping downloads')

	sys.exit()

if __name__ == "__main__":
   main(sys.argv[1:])