Initial Commit

.gitignore

@@ -0,0 +1,46 @@
+# Git ignore File
+.*
+*.pyc
+*/*.pyc
+*/*/*.pyc
+*/*/*/*.pyc
+*/*/*/*/*.pyc
+*/__pycache__
+*/*/__pycache__
+*/*/*/__pycache__
+!/.gitignore
+*.directory
+*/*.directory
+*/*/*.directory
+*/*/*/*.directory
+*/*/*/*/*.directory
+data.csv
+*/data.csv
+*/*/data.csv
+*/*/*/data.csv
+*.log
+*/*.log
+*/*/*.log
+*/*/*/*.log
+*/*/*/*/*.log
+*.out
+*/*.out
+*/*/*.out
+*/*/*/*.out
+*/*/*/*/*.out
+*.ydc
+*/*.ydc
+*/*/*.ydc
+*/*/*/*.ydc
+*/*/*/*/*.ydc
+*.html
+*/*.html
+*/*/*.html
+*/*/*/*.html
+*/*/*/*/*.html
+*.pdf
+*/*.pdf
+*/*/*.pdf
+*/*/*/*.pdf
+*/*/*/*/*.pdf
+*.idea

GenerateRandomCSV.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python2
+import sys
+from random import randint
+print "FIELD1,FIELD2,FIELD3,FIELD4,FIELD5,FIELD6,FIELD7,FIELD8,FIELD9,FIELD10"
+for i in range(0,1000000):
+    print "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s" %(randint(0,sys.maxint),randint(0,sys.maxint),randint(0,sys.maxint),randint(0,sys.maxint),randint(0,sys.maxint),randint(0,sys.maxint),randint(0,sys.maxint),randint(0,sys.maxint),randint(0,sys.maxint),randint(0,sys.maxint),)

MakeDebianIso.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+__author__ = 'Andreas Bader'
+__version__ = "0.01"
+
+import argparse
+import logging
+import Util
+import os
+import shutil
+from fabric.api import *
+
+outputFileNameSuffix="-autoinstall"
+
+neededTools = ["which", "sed","7z", "genisoimage", "gzip", "find", "cpio", "md5sum"]
+
+# Configure ArgumentParser
+parser = argparse.ArgumentParser(prog="MakeDebianIso.py",version=__version__,description="Bla", formatter_class=argparse.RawDescriptionHelpFormatter, epilog="")
+parser.add_argument("-t", "--tmpfolder", metavar="TMPFOLDER", required=True, help="path to tmp space (> 5 gb)")
+parser.add_argument("-i", "--isofile", metavar="ISOFILE", required=True, help="path to Debian Iso")
+parser.add_argument("-f", "--outputfolder", metavar="OUTPUTFOLDER", required=True, help="path to outputfolder (will be created if not existent)")
+parser.add_argument("-p", "--presseedfile", metavar="PRESEEDFILE", required=True, help="path to preseedfile")
+parser.add_argument("-o", "--overwrite", action="store_true", help="overwrite if ISO is existing")
+parser.add_argument("-l", "--log", action='store_true', help="Be more verbose, log vagrant output.")
+args = parser.parse_args()
+
+# Configure Logging
+logLevel = logging.WARN
+if args.log:
+    logLevel = logging.DEBUG
+logging.basicConfig(level=logLevel)
+logger = logging.getLogger(__name__)
+
+# Check ob alle Tools da sind
+for tool in neededTools:
+    with settings(warn_only=True), hide('output','running','warnings'):
+        ret=local("which '%s'" %(tool))
+        if ret.return_code != 0:
+            logger.error("'%s' not found, please install." %(tool))
+            exit(-1)
+
+if not (Util.check_file_exists(args.presseedfile) and Util.check_file_readable(args.presseedfile)):
+    logger.error("Preseedfile %s does not exist or is not readable." %(args.presseedfile))
+    exit(-1)
+
+if not (Util.check_file_exists(args.isofile) and Util.check_file_readable(args.isofile)):
+    logger.error("Isofile %s does not exist or is not readable." %(args.isofile))
+    exit(-1)
+
+if not Util.check_folder(args.outputfolder,logger,False,True):
+    if not Util.create_folder(args.outputfolder) or not Util.check_folder(args.outputfolder,logger):
+        logger.error("Can't create %s." %(args.outputfolder))
+        exit(-1)
+
+if not Util.check_folder(args.tmpfolder,logger,False,True):
+    if not Util.create_folder(args.tmpfolder) or not Util.check_folder(args.tmpfolder,logger):
+        logger.error("Can't create %s." %(args.tmpfolder))
+        exit(-1)
+else:
+    Util.delete_folder(args.tmpfolder,logger)
+    if not Util.create_folder(args.tmpfolder) or not Util.check_folder(args.tmpfolder,logger):
+        logger.error("Can't create %s." %(args.tmpfolder))
+        exit(-1)
+
+isoFileName=os.path.basename(args.isofile)
+isoFileBase="default"
+if len(os.path.splitext(args.isofile)) > 0:
+    isoFileBase=os.path.splitext(isoFileName)[0]
+isoFileExt=".iso"
+if len(os.path.splitext(args.isofile)) > 1:
+    isoFileExt=os.path.splitext(isoFileName)[1]
+preseedSuffix=""
+if len(os.path.splitext(os.path.basename(args.presseedfile))[0].split("-")) > 1:
+    for suffix in os.path.splitext(os.path.basename(args.presseedfile))[0].split("-")[1:]:
+        preseedSuffix += "-%s" %(suffix)
+newIsoFileName="%s%s%s%s" %(isoFileBase,preseedSuffix,outputFileNameSuffix,isoFileExt)
+isoPath=os.path.join(args.tmpfolder,isoFileName)
+
+if Util.check_file_exists(os.path.join(args.outputfolder,newIsoFileName)):
+    if args.overwrite:
+        if not Util.delete_file(os.path.join(args.outputfolder,newIsoFileName),logger):
+            logger.error("Error while deleting %s." %(os.path.join(args.outputfolder,newIsoFileName)))
+            exit(-1)
+    else:
+        logger.error("Outputfile %s does exist." %(os.path.join(args.outputfolder,newIsoFileName)))
+        exit(-1)
+
+# Copy Iso to tmpfolder
+try:
+    shutil.copy(args.isofile, isoPath)
+except Exception, e:
+    logger.error("Can't copy %s to %s." %(args.isofile,args.tmpfolder), exc_info=True)
+    exit(-1)
+
+# Unpack it with 7z
+with lcd(args.tmpfolder), settings(warn_only=True), hide('output','running','warnings'):
+    ret=local("7z x '%s'" %(isoFileName), capture=True)
+    if not ret.succeeded:
+        logger.error("Can't unpack %s. Error: %s" %(isoPath,ret.stderr))
+        exit(-1)
+
+# Delete ISO
+if not Util.delete_file(isoPath, logger):
+    logger.error("Can't delete %s." %(isoPath))
+    exit(-1)
+
+# mkdir irmod, unzip initrd.gz
+if not Util.create_folder(os.path.join(args.tmpfolder,"irmod")):
+    logger.error("Can't create %s." %(os.path.join(args.tmpfolder,"irmod")))
+    exit(-1)
+with lcd(os.path.join(args.tmpfolder,"irmod")), settings(warn_only=True), hide('output','running','warnings'):
+    ret=local("gzip -d < ../install.amd/initrd.gz | cpio --extract --verbose --make-directories --no-absolute-filenames")
+    if ret.return_code != 2:
+        logger.error("Can't unpack %s. Error: %s" %(os.path.join(args.tmpfolder,"install.amd/initrd.gz"),ret.stderr))
+        exit(-1)
+
+# Copy Pressed file in place
+if not Util.copy_file(args.presseedfile, os.path.join(args.tmpfolder,"irmod","preseed.cfg"),logger):
+    logger.error("Can't copy %s to %s." %(args.presseedfile, os.path.join(args.tmpfolder,"irmod","preseed.cfg")))
+    exit(-1)
+
+# Pack & Compress it again
+with lcd(os.path.join(args.tmpfolder,"irmod")), settings(warn_only=True), hide('output','running','warnings'):
+    ret=local("find . | cpio -H newc --create --verbose | gzip -9 > ../install.amd/initrd.gz")
+    if not ret.succeeded:
+        logger.error("Can't pack %s. Error: %s" %(os.path.join(args.tmpfolder,"install.amd/initrd.gz"),ret.stderr))
+        exit(-1)
+
+# Delete irmod
+if not Util.delete_folder(os.path.join(args.tmpfolder,"irmod"),logger):
+    logger.error("Can't delete %s." %(os.path.join(args.tmpfolder,"irmod")))
+    exit(-1)
+
+# Running sed + generating md5sum + genisoimage
+with lcd(args.tmpfolder), settings(warn_only=True), hide('output','running','warnings'):
+    ret=local("sed -i \"s/timeout 0/timeout 2/g\" isolinux/isolinux.cfg")
+    if not ret.succeeded:
+        logger.error("Error while sed run on %s. Error: %s" %(os.path.join(args.tmpfolder,"isolinux/isolinux.cfg"),ret.stderr))
+        exit(-1)
+    ret=local("sed -i \"s/quiet/ipv6.disable=1 quiet/g\" isolinux/txt.cfg")
+    if not ret.succeeded:
+        logger.error("Error while sed run on %s. Error: %s" %(os.path.join(args.tmpfolder,"isolinux/txt.cfg"),ret.stderr))
+        exit(-1)
+    ret=local("md5sum `find -follow -type f` > md5sum.txt")
+    if not ret.succeeded:
+        logger.error("Error while md5sum+find run. Error: %s" %(ret.stderr))
+        exit(-1)
+    ret=local("genisoimage -o \"%s\" -r -J -no-emul-boot -boot-load-size 4 -boot-info-table -b isolinux/isolinux.bin -c isolinux/boot.cat ." %(os.path.join(args.tmpfolder,newIsoFileName)))
+    if not ret.succeeded:
+        logger.error("Error while genisoimage run. Error: %s" %(ret.stderr))
+        exit(-1)
+
+# Copy Iso, delete tmpfolder
+if os.path.join(args.tmpfolder,newIsoFileName) != os.path.join(args.outputfolder,newIsoFileName):
+    if not Util.copy_file(os.path.join(args.tmpfolder,newIsoFileName),os.path.join(args.outputfolder,newIsoFileName),logger):
+        logger.error("Could not copy %s to %s." %(os.path.join(args.tmpfolder,newIsoFileName),os.path.join(args.outputfolder,newIsoFileName)))
+        exit(-1)
+if not Util.delete_folder(args.tmpfolder,logger):
+    logger.error("Can't delete %s." %(args.tmpfolder))
+    exit(-1)
+logger.info("Done.")
+exit(0)

MakeDebianQcow2.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+__author__ = 'Andreas Bader'
+__version__ = "0.01"
+
+import argparse
+import logging
+import Util
+import os
+import shutil
+from fabric.api import *
+
+outputFileNameSuffix="-autoinstall"
+
+neededTools = ["which", "qemu-img","kvm"]
+
+# Configure ArgumentParser
+parser = argparse.ArgumentParser(prog="MakeDebianQcow2.py",version=__version__,description="Bla", formatter_class=argparse.RawDescriptionHelpFormatter, epilog="")
+parser.add_argument("-i", "--isofile", metavar="ISOFILE", required=True, help="path to Debian Autoinstall Iso")
+parser.add_argument("-f", "--outputfolder", metavar="OUTPUTFOLDER", required=True, help="path to outputfolder (will be created if not existent) (> 5 gb)")
+parser.add_argument("-g", "--graphic", action="store_true", help="show graphic output, normally -nographic is used. Useful if you don't use a preseeded iso.")
+parser.add_argument("-o", "--overwrite", action="store_true", help="overwrite if QCOW2 File is existing")
+parser.add_argument("-l", "--log", action='store_true', help="Be more verbose, log vagrant output.")
+parser.add_argument("-s", "--size", metavar="SIZE", type=int, default=50, help="Size in Gigabyte (50 is default), e.g. 50 for 50G (will be compressed, Qcow2 file size will be much lesser!)")
+args = parser.parse_args()
+
+# Configure Logging
+logLevel = logging.WARN
+if args.log:
+    logLevel = logging.DEBUG
+logging.basicConfig(level=logLevel)
+logger = logging.getLogger(__name__)
+
+# Check ob alle Tools da sind
+for tool in neededTools:
+    with settings(warn_only=True), hide('output','running','warnings'):
+        ret=local("which '%s'" %(tool))
+        if ret.return_code != 0:
+            logger.error("'%s' not found, please install." %(tool))
+            exit(-1)
+
+if not (Util.check_file_exists(args.isofile) and Util.check_file_readable(args.isofile)):
+    logger.error("Isofile %s does not exist or is not readable." %(args.isofile))
+    exit(-1)
+
+if not Util.check_folder(args.outputfolder,logger,False,True):
+    if not Util.create_folder(args.outputfolder) or not Util.check_folder(args.outputfolder,logger):
+        logger.error("Can't create %s." %(args.outputfolder))
+        exit(-1)
+
+isoFileName=os.path.basename(args.isofile)
+imgFileBase="default"
+if len(os.path.splitext(args.isofile)) > 0:
+    imgFileBase=os.path.splitext(isoFileName)[0]
+imgFileExtRaw=".img"
+imgFileExtQcow2=".qcow2"
+imgFileRaw="%s%s" %(imgFileBase,imgFileExtRaw)
+imgFileQcow2="%s%s" %(imgFileBase, imgFileExtQcow2)
+
+for file in [imgFileRaw,imgFileQcow2]:
+    if Util.check_file_exists(os.path.join(args.outputfolder,file)):
+        if args.overwrite:
+            if not Util.delete_file(os.path.join(args.outputfolder,file),logger):
+                logger.error("Error while deleting %s." %(os.path.join(args.outputfolder,file)))
+                exit(-1)
+        else:
+            logger.error("Outputfile %s does exist." %(os.path.join(args.outputfolder,file)))
+            exit(-1)
+
+with lcd(os.path.join(args.outputfolder)), settings(warn_only=True), hide('output','running','warnings'):
+    ret=local("qemu-img create -f qcow2 \"%s\" %sG" %(imgFileRaw,args.size))
+    if ret.return_code != 0:
+        logger.error("Can't create %s. Error: %s" %(os.path.join(args.outputfolder,imgFileRaw),ret.stderr))
+        exit(-1)
+    graphic=" -nographic"
+    if args.graphic:
+        graphic=""
+    ret=local("kvm -hda \"%s\" -cdrom \"%s\" -m 2048 -smp 2%s" %(imgFileRaw,args.isofile,graphic))
+    if ret.return_code != 0:
+        logger.error("Can't install %s on %s with kvm. Error: %s" %(args.isofile,os.path.join(args.outputfolder,imgFileRaw),ret.stderr))
+        exit(-1)
+    # compat needed if NOT using an very old qemu-img!
+    ret=local("qemu-img convert -f qcow2 -c \"%s\" -o compat=0.10 -O qcow2 \"%s\"" %(imgFileRaw,imgFileQcow2))
+    if ret.return_code != 0:
+        logger.error("Can't compress %s to %s. Error: %s" %(os.path.join(args.outputfolder,imgFileRaw),os.path.join(args.outputfolder,imgFileQcow2),ret.stderr))
+        exit(-1)
+
+if not Util.delete_file(os.path.join(args.outputfolder,imgFileRaw), logger):
+    logger.error("Can't delete %s." %(os.path.join(args.outputfolder,imgFileRaw)))
+    exit(-1)
+exit(0)