openmesh/CI/gitlab-ci/assemble-gitlab-ci.py

#!/usr/bin/python3

import sys, os, re, yaml, hashlib

# Version 3.2

# Script for automated gitlab-ci creation
# Assembles the gitlab ci from master template file:
master_file = 'ci-master.yml'
# Lines in the master file are copied to the resulting
# assemblied gitlab ci file
target_file = '../../.gitlab-ci.yml'
# Lines that are {xxx} Strings are interpreted
# as import statement. Therefore the file xxx is imported
# into that line.
# Lines that are {xxx,option1=...,option2=...} includes
# the file xxx but replaces {{option1}} etc with specified
# string.
error_on_path_redirection = True
# Notice that xxx can not contain path redirections
# like .. and /

# Max import recursion
maxFileRecursionDepth = 4
# Max filename used for pretty print
maxFilnameChars = 30


# Prefix to prepend to master file
autogenerated_notice = """#############################################################
#                                                           #
# This is an auto generated file. Do not make               #
# changes to this file. They possible will be overriden.    #
#                                                           #
# To make persistent changes, changes files in              #
# ./CI/gitlab-ci/ ...                                       #
# and regenerate this file with the configuration tool      #
# python3 ./CI/gitlab-ci/assemble-gitlab-ci.py              #
#                                                           #
#############################################################

"""


# Checks if an import filename is valid - free of path redirections
def isValidImportFilename(filenameToImport):
    if not error_on_path_redirection:
        return True
    else:
        filterRegex = r"(\/|\\|\.\.+)"
        filtered = re.sub(filterRegex, '', filenameToImport)
        return filenameToImport == filtered

# Returns the directory to work on
def findCIAssemblyDirectory():
    pathname = os.path.dirname(sys.argv[0])      
    return os.path.abspath(pathname)

# Returns file content as string
def readFile(filename):
    file = open(filename, "r")
    content = file.read()
    file.close()
    return content

# Parse File Import String for variable replacements
def fetchVariableReplacers(variablesGrep):
    if (variablesGrep == None):
        return {}

    regex_option = r"([^\}\n\=,]+)\=([^\}\n\=,]+)"
    pattern = re.compile(regex_option, flags=re.MULTILINE)
    result = {}

    for (key, value) in re.findall(pattern, variablesGrep):

        if (key != None and value != None):
            key = key.strip()
            result[key] = value

    return result


# Assembles the file in memory and returns file content as string
def assembleTarget(master, depth=maxFileRecursionDepth):
    if depth < 0:
        raise "Max depth reached. Possible circular import?"
    print_prefix = ""
    for _ in range(0, maxFileRecursionDepth-depth):
        print_prefix = print_prefix + " | \t"
    print_prefix_inverse = ""
    for _ in range(0, depth):
        print_prefix_inverse = print_prefix_inverse + "\t"

    master_content = readFile(master)
    regex_import_stmt = r"^\ *\{([^\},\n]+)(,[^=\n\}\,]+\=[^\}\n,]*)*\}\ *$"
    regex_import_comp = re.compile(regex_import_stmt)
    master_content_list = master_content.splitlines()

    # Walk through file looking for import statements
    cur_index = 0
    while cur_index < len(master_content_list):
        cur_line = master_content_list[cur_index]
        match = regex_import_comp.match(cur_line)

        if match:
            importFile = match.groups()[0]
            if importFile:
                # Found import statement
                print(print_prefix+"Importing file: "+importFile.ljust(maxFilnameChars), end="")

                if not isValidImportFilename(importFile):
                    raise "Invalid filename "+importFile+ ". Do not include path redirections"
                
                variablesGrep = match.string
                variableReplacers = fetchVariableReplacers(variablesGrep)

                print(print_prefix_inverse, variableReplacers)

                import_content = assembleTarget(importFile, depth=depth-1)

                for key, value in variableReplacers.items():
                    import_content = import_content.replace(r"{{"+key+r"}}", value)

                import_content_list = import_content.splitlines()
                master_content_list.pop(cur_index)
                for new_line in reversed(import_content_list):
                    master_content_list.insert(cur_index, new_line)

        cur_index += 1

    # Assemble result
    master_content = ''.join(str(e)+'\n' for e in master_content_list)
    return master_content

# Main function
def main():
    print("Starting config assembly")
    os.chdir(findCIAssemblyDirectory())
    target_content = autogenerated_notice
    target_content += assembleTarget(master_file)

    m = hashlib.sha256()
    m.update(readFile(target_file).encode('utf-8'))
    hash_original = m.hexdigest()
    m = hashlib.sha256()
    m.update(target_content.encode('utf-8'))
    m.update("\n".encode('utf-8'))
    hash_new = m.hexdigest()

    print("Old checksum: ", hash_original)
    print("New checksum: ", hash_new)

    if (hash_original == hash_new):
        print("No changes made: Skipping file write")
    else:
        print("File differs")
        print("Writing config to file "+target_file)
        target_file_handle = open(target_file, "w")
        target_file_handle.write(target_content)
        target_file_handle.write("\n")
        target_file_handle.flush()
        target_file_handle.close()

    try:
        yaml.load(target_content, Loader=yaml.SafeLoader)
        print("Yaml syntax check: OK")
    except Exception as e:
        print("Invalid yaml syntax:", e)

    print("Finished.")


# Execute main function
if __name__ == '__main__':
    main()