CompressHtml.py

Import("env")
#########################################################################################################################################################
#
# CompressHtml
#
# This script is intended for joining a html-file with its js & css files and then compress it and convert it to Arduino (ESP32/ESP8266) PROGMEM format
#
# It will release you from the burden of uploading a FS-Image or convert html back and forth.
#
# Pro`s:
# - One-compile run
# - One source of html, which can be debugged by using live-server, so before even uploading it
# - converges html with its javascript and css files and then compresses it for even faster loading
# - Fast & compact, expect compress rates 2-4x as small, for html/js/css.
# - Also other/multiple files can be converted
# - leaves SPIFFS or LittleFs on the ESP intact, so stored data will not be touched.
# - crude //REMOVE  //ENDREMOVE build in for debugging js, this will be stripped by the script.
#
# Cons:
# - takes up progmem space
# - static, not a problem in most cases though
# - little need to setup
#
# Setup:
#
# You`ll need:
# - platformIO
# - this script
# - Compress.yml
# - for serving files, refer to the imagenames of the files in include/appdata.h
#
# Of course you also could do other things with this like a large lookup table.
# Use 'live-server' extension to serve your webpage in your html folder from localhost. With some tricks you can
# call to your ESP-api or ws connection then. This will save you compile and upload time when working on your ESP-s webpage.

import os, gzip, re
from datetime import datetime

try:
    import yaml
except ImportError:
    env.Execute("$PYTHONEXE -m pip install pyyaml")

try:
    import markdown
except ImportError:
    env.Execute("$PYTHONEXE -m pip install Markdown")


def GetProgMemString(data, progmemName, addendum = "", comment = ""):
    buffer = f' \n\n// {comment}\nconst uint8_t {progmemName + addendum}[] PROGMEM = {{\n'
    bytecount = 0
    for b in data:
        buffer += "0x%02x" % b
        if (bytecount != len(data)-1):
            buffer += ", "
        bytecount +=1
        if ((bytecount % 64) == 0):
            buffer += "\n"
    buffer += "\n};"
    return buffer

def SubstituteCssLinks(htmlData, basedir):
    cssLinks = re.findall("<link .+?>", htmlData)
    for cssLink in cssLinks:
        print (f" add css-link: {cssLink}")
        cssFilePath = f'{basedir}/' + re.findall("href=\"(.+?)\"",cssLink)[0]
        cssFile = open(cssFilePath, "r")
        cssFileContent = cssFile.read()
        replaceCssLinkBuffer = "\n<style>\n"
        replaceCssLinkBuffer += cssFileContent
        replaceCssLinkBuffer += "\n</style>\n"
        htmlData = htmlData.replace(cssLink, replaceCssLinkBuffer)
    return htmlData

def SubstituteJsLinks(htmlData, basedir):
    jsLinks = re.findall("<script.+?src=\".+?.js\"></script>", htmlData)
    for jsLink in jsLinks:
        print (f" add js-link: {jsLink}")
        jsFilePath = f'{basedir}/' + re.findall(" src=\"(.+?)\"",jsLink)[0]
        jsFile = open(jsFilePath, "r")
        jsFileContent = jsFile.read()
        replaceJsLinkBuffer = "\n<script>\n"
        replaceJsLinkBuffer += jsFileContent
        replaceJsLinkBuffer += "\n</script>\n"
        htmlData = htmlData.replace(jsLink, replaceJsLinkBuffer)
    return htmlData

def RemoveJSRemoves(htmlData):
    toRemoves = jsLinks = re.findall("\/\/REMOVE.+?\/\/ENDREMOVE", htmlData, re.DOTALL) 
    for toRemove in toRemoves:
        htmlData = htmlData.replace(toRemove,"")
    return htmlData

def ReadAndCreateGetProgMemString(filename):
        if (os.path.isfile(filename) == False):
            print(f"\nFile: {filename} was not found...\n")
            return ""
        inFile = open(filename, "rb")
        inData = inFile.read()
        return GetProgMemString(inData, filename.split("/")[-1].replace(".","_"))

def GetTxtFile(filename):
    print (f'processing file: {filename}')
    with open(filename, "r") as file:
        return file.read()
    
def GetBinFile(filename):
    print (f'processing file: {filename}')
    with open(filename, "rb") as file:
        return file.read()

def FileHeader():
    header = f'// This file is auto-generated by CompressHtml.py @{datetime.now().strftime("%d/%m/%Y %H:%M:%S")}\n'
    header += "// Do not edit, since it will be overwritten\n\n" 
    header += "#include <Arduino.h>\n"
    return header

def GzCompressText(data):
    outData = gzip.compress(bytes(data, "UTF-8"))
    return outData

def GzCompressBin(data):
    return gzip.compress(data)

def ProgMemNameFromFileName(filename):
    return filename.split("/")[-1].replace(".","_")

def processHtml(filename, basedir):
    HtmlData = GetTxtFile(filename)
    HtmlData = SubstituteJsLinks(HtmlData, basedir)
    HtmlData = RemoveJSRemoves(HtmlData)
    HtmlData = SubstituteCssLinks(HtmlData, basedir)
    BinGzHtmlData = GzCompressText(HtmlData)
    inSize = len(HtmlData)
    outSize = len(BinGzHtmlData)
    print(f" Size in: {inSize}, size out: {outSize}, ratio: {round(outSize*100/inSize,1)}%.")
    return GetProgMemString(BinGzHtmlData,ProgMemNameFromFileName(filename + "_gz"))

def processMD(filename):
    mdData = GetTxtFile(filename)
    html_md_content = markdown.markdown(mdData)
    BinGzMdData = GzCompressText(html_md_content)
    inSize = len(html_md_content)
    outSize = len(BinGzMdData)
    print(f" Size in: {inSize}, size out: {outSize}, ratio: {round(outSize*100/inSize,1)}%.")
    return GetProgMemString(BinGzMdData,ProgMemNameFromFileName(filename + "_html_gz"))

def processOtherGz(filename):
    BinData = GetBinFile(filename)
    BinGzData = GzCompressBin(BinData)
    inSize = len(BinData)
    outSize = len(BinGzData)
    print(f" Size in: {inSize}, size out: {outSize}, ratio: {round(outSize*100/inSize,1)}%.")
    return GetProgMemString(BinGzData,ProgMemNameFromFileName(filename + "_gz"))

def ProcessOther(filename):
    BinData = GetBinFile(filename)
    print(f" Size in: {len(BinData)} (no compression).")
    return GetProgMemString(BinData,ProgMemNameFromFileName(filename))


# main:

print("\n**************** Compress.py start ****************\n")

with open('Compress.yml', 'r') as file:
    ymlConfig = yaml.safe_load(file)

print("Config: Compress.yml was red.")

headerFileData = FileHeader()

for file in ymlConfig["HTML"]:
    headerFileData += processHtml(ymlConfig["SOURCEDIR"] + "/" + file, ymlConfig["SOURCEDIR"])

for file in ymlConfig["MARKDOWN"]:
    headerFileData += processMD(file)

for file in ymlConfig["OTHERSGZ"]:
    headerFileData += processOtherGz(ymlConfig["SOURCEDIR"] + "/" + file)

for file in ymlConfig["OTHERS"]:
    headerFileData += ProcessOther(ymlConfig["SOURCEDIR"] + "/" + file)

with open(ymlConfig["DESTINATIONFILE"], "w") as outfile:
    outfile.write(headerFileData)

print("\n**************** Compress.py end ****************\n")