#!/usr/bin/python3 import sys, os, re, yaml, hashlib # Version 3.3 # Script for automated gitlab-ci creation # Assembles the gitlab ci from master template file: master_file = 'ci-master.yml' # Lines in the master file are copied to the resulting # assemblied gitlab ci file target_file = '../../.gitlab-ci.yml' # Lines that are {xxx} Strings are interpreted # as import statement. Therefore the file xxx is imported # into that line. # Lines that are {xxx,option1=...,option2=...} includes # the file xxx but replaces {{option1}} etc with specified # string. error_on_path_redirection = True # Notice that xxx can not contain path redirections # like .. and / # Max import recursion maxFileRecursionDepth = 4 # Max filename used for pretty print maxFilnameChars = 30 # Prefix to prepend to master file autogenerated_notice = """############################################################# # # # This is an auto generated file. Do not make # # changes to this file. They possible will be overriden. # # # # To make persistent changes, changes files in # # ./CI/gitlab-ci/ ... # # and regenerate this file with the configuration tool # # python3 ./CI/gitlab-ci/assemble-gitlab-ci.py # # # ############################################################# """ # Checks if an import filename is valid - free of path redirections def isValidImportFilename(filenameToImport): if not error_on_path_redirection: return True else: filterRegex = r"(\/|\\|\.\.+)" filtered = re.sub(filterRegex, '', filenameToImport) return filenameToImport == filtered # Returns the directory to work on def findCIAssemblyDirectory(): pathname = os.path.dirname(sys.argv[0]) return os.path.abspath(pathname) # Returns file content as string def readFile(filename): file = open(filename, "r") content = file.read() file.close() return content # Parse File Import String for variable replacements def fetchVariableReplacers(variablesGrep): if (variablesGrep == None): return {} regex_option = r"([^\}\n\=,]+)\=([^\}\n\=,]+)" pattern = re.compile(regex_option, flags=re.MULTILINE) result = {} for (key, value) in re.findall(pattern, variablesGrep): if (key != None and value != None): key = key.strip() result[key] = value.strip() return result # Assembles the file in memory and returns file content as string def assembleTarget(master, depth=maxFileRecursionDepth): if depth < 0: raise "Max depth reached. Possible circular import?" print_prefix = "" for _ in range(0, maxFileRecursionDepth-depth): print_prefix = print_prefix + " | \t" print_prefix_inverse = "" for _ in range(0, depth): print_prefix_inverse = print_prefix_inverse + "\t" master_content = readFile(master) regex_import_stmt = r"^\ *\{([^\},\n]+)(,[^=\n\}\,]+\=[^\}\n,]*)*\}\ *$" regex_import_comp = re.compile(regex_import_stmt) master_content_list = master_content.splitlines() # Walk through file looking for import statements cur_index = 0 while cur_index < len(master_content_list): cur_line = master_content_list[cur_index] match = regex_import_comp.match(cur_line) if match: importFile = match.groups()[0] if importFile: # Found import statement print(print_prefix+"Importing file: "+importFile.ljust(maxFilnameChars), end="") if not isValidImportFilename(importFile): raise "Invalid filename "+importFile+ ". Do not include path redirections" variablesGrep = match.string variableReplacers = fetchVariableReplacers(variablesGrep) print(print_prefix_inverse, variableReplacers) import_content = assembleTarget(importFile, depth=depth-1) for key, value in variableReplacers.items(): import_content = import_content.replace(r"{{"+key+r"}}", value) import_content_list = import_content.splitlines() master_content_list.pop(cur_index) for new_line in reversed(import_content_list): master_content_list.insert(cur_index, new_line) cur_index += 1 # Assemble result master_content = ''.join(str(e)+'\n' for e in master_content_list) return master_content # Main function def main(): print("Starting config assembly") os.chdir(findCIAssemblyDirectory()) target_content = autogenerated_notice target_content += assembleTarget(master_file) m = hashlib.sha256() m.update(readFile(target_file).encode('utf-8')) hash_original = m.hexdigest() m = hashlib.sha256() m.update(target_content.encode('utf-8')) m.update("\n".encode('utf-8')) hash_new = m.hexdigest() print("Old checksum: ", hash_original) print("New checksum: ", hash_new) if (hash_original == hash_new): print("No changes made: Skipping file write") else: print("File differs") print("Writing config to file "+target_file) target_file_handle = open(target_file, "w") target_file_handle.write(target_content) target_file_handle.write("\n") target_file_handle.flush() target_file_handle.close() try: yaml.load(target_content, Loader=yaml.SafeLoader) print("Yaml syntax check: OK") except Exception as e: print("Invalid yaml syntax:", e) print("Finished.") # Execute main function if __name__ == '__main__': main()