Merge branch 'docker-build-system' into 'master'

Docker build system

See merge request !234

.gitlab-ci.yml

@@ -6,6 +6,7 @@
 # To make persistent changes changes files in               #
 # ./CI/gitlab-ci/ ...                                       #
 # and regenerate this file with the configuration tool      #
+# python3 ./CI/gitlab-ci/assemble-gitlab-ci.py              #
 #                                                           #
 #############################################################
 
@@ -25,138 +26,236 @@ cppcheck:
   stage: build
   script: "CI/ci-cppcheck.sh"
   image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker] 
+  tags: [Docker]
+  timeout: 3h
   artifacts:
     paths:
       - cppcheck.log
 
-gcc-c++11-test-debug:
-  stage: test
-  script: "CI/ci-linux-test.sh gcc C++11 debug"
+#-----------  Job Informations:
+# Type:       Build: debug
+# OS:         Linux
+# Compiler:   gcc
+# Language:   cpp11
+
+build-debug-gcc-cpp11:
+  stage: build
+  script: "CI/ci-linux-build.sh gcc cpp11 debug"
   image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
   tags: [Docker]
-  dependencies: [gcc-c++11-build-debug]
+  artifacts:
+    paths:
+    - build-gcc-cpp11-debug-Vector-Checks/
 
-gcc-c++11-test-release:
+#-----------  Job Informations:
+# Type:       Test: debug
+# OS:         Linux
+# Compiler:   gcc
+# Language:   cpp11
+
+test-debug-gcc-cpp11:
   stage: test
-  script: "CI/ci-linux-test.sh gcc C++11 release"
+  script: "CI/ci-linux-test.sh gcc cpp11 debug"
   image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
   tags: [Docker]
-  dependencies: [gcc-c++11-build-release]
+  dependencies: [build-debug-gcc-cpp11]
+
+#-----------  Job Informations:
+# Type:       Build: release
+# OS:         Linux
+# Compiler:   gcc
+# Language:   cpp11
 
-gcc-c++11-build-debug:
+build-release-gcc-cpp11:
   stage: build
-  script: "CI/ci-linux-build.sh gcc C++11 debug"
+  script: "CI/ci-linux-build.sh gcc cpp11 release"
   image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
   tags: [Docker]
   artifacts:
     paths:
-    - build-debug-gcc-cpp11-Vector-Checks/
+    - build-gcc-cpp11-release-Vector-Checks/
+
+#-----------  Job Informations:
+# Type:       Test: release
+# OS:         Linux
+# Compiler:   gcc
+# Language:   cpp11
 
-gcc-c++11-build-release:
+test-release-gcc-cpp11:
+  stage: test
+  script: "CI/ci-linux-test.sh gcc cpp11 release"
+  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
+  tags: [Docker]
+  dependencies: [build-release-gcc-cpp11]
+
+#-----------  Job Informations:
+# Type:       Build: debug
+# OS:         Linux
+# Compiler:   clang
+# Language:   cpp11
+
+build-debug-clang-cpp11:
   stage: build
-  script: "CI/ci-linux-build.sh gcc C++11 release"
+  script: "CI/ci-linux-build.sh clang cpp11 debug"
   image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
   tags: [Docker]
   artifacts:
     paths:
-    - build-release-gcc-cpp11-Vector-Checks/
+    - build-clang-cpp11-debug-Vector-Checks/
 
-clang-c++11-test-debug:
-  stage: test
-  script: "CI/ci-linux-test.sh clang C++11 debug"
-  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker]
-  dependencies: [clang-c++11-build-debug]
+#-----------  Job Informations:
+# Type:       Test: debug
+# OS:         Linux
+# Compiler:   clang
+# Language:   cpp11
 
-clang-c++11-test-release:
+test-debug-clang-cpp11:
   stage: test
-  script: "CI/ci-linux-test.sh clang C++11 release"
+  script: "CI/ci-linux-test.sh clang cpp11 debug"
   image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
   tags: [Docker]
-  dependencies: [clang-c++11-build-release]
+  dependencies: [build-debug-clang-cpp11]
+
+#-----------  Job Informations:
+# Type:       Build: release
+# OS:         Linux
+# Compiler:   clang
+# Language:   cpp11
 
-clang-c++11-build-debug:
+build-release-clang-cpp11:
   stage: build
-  script: "CI/ci-linux-build.sh clang C++11 debug"
+  script: "CI/ci-linux-build.sh clang cpp11 release"
   image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
   tags: [Docker]
   artifacts:
     paths:
-    - build-debug-clang-cpp11-Vector-Checks/
+    - build-clang-cpp11-release-Vector-Checks/
+
+#-----------  Job Informations:
+# Type:       Test: release
+# OS:         Linux
+# Compiler:   clang
+# Language:   cpp11
+
+test-release-clang-cpp11:
+  stage: test
+  script: "CI/ci-linux-test.sh clang cpp11 release"
+  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
+  tags: [Docker]
+  dependencies: [build-release-clang-cpp11]
 
-clang-c++11-build-release:
+#-----------  Job Informations:
+# Type:       Build: debug
+# OS:         Linux
+# Compiler:   gcc
+# Language:   cpp14
+
+build-debug-gcc-cpp14:
   stage: build
-  script: "CI/ci-linux-build.sh clang C++11 release"
+  script: "CI/ci-linux-build.sh gcc cpp14 debug"
   image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker] 
+  tags: [Docker]
   artifacts:
     paths:
-    - build-release-clang-cpp11-Vector-Checks/
+    - build-gcc-cpp14-debug-Vector-Checks/
 
-gcc-c++14-test-debug:
-  stage: test
-  script: "CI/ci-linux-test.sh gcc C++14 debug"
-  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker]
-  dependencies: [gcc-c++14-build-debug]
+#-----------  Job Informations:
+# Type:       Test: debug
+# OS:         Linux
+# Compiler:   gcc
+# Language:   cpp14
 
-gcc-c++14-test-release:
+test-debug-gcc-cpp14:
   stage: test
-  script: "CI/ci-linux-test.sh gcc C++14 release"
+  script: "CI/ci-linux-test.sh gcc cpp14 debug"
   image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
   tags: [Docker]
-  dependencies: [gcc-c++14-build-release]
+  dependencies: [build-debug-gcc-cpp14]
+
+#-----------  Job Informations:
+# Type:       Build: release
+# OS:         Linux
+# Compiler:   gcc
+# Language:   cpp14
 
-gcc-c++14-build-debug:
+build-release-gcc-cpp14:
   stage: build
-  script: "CI/ci-linux-build.sh gcc C++14 debug"
+  script: "CI/ci-linux-build.sh gcc cpp14 release"
   image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
   tags: [Docker]
   artifacts:
     paths:
-    - build-debug-gcc-cpp14-Vector-Checks/
+    - build-gcc-cpp14-release-Vector-Checks/
+
+#-----------  Job Informations:
+# Type:       Test: release
+# OS:         Linux
+# Compiler:   gcc
+# Language:   cpp14
+
+test-release-gcc-cpp14:
+  stage: test
+  script: "CI/ci-linux-test.sh gcc cpp14 release"
+  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
+  tags: [Docker]
+  dependencies: [build-release-gcc-cpp14]
 
-gcc-c++14-build-release:
+#-----------  Job Informations:
+# Type:       Build: debug
+# OS:         Linux
+# Compiler:   clang
+# Language:   cpp14
+
+build-debug-clang-cpp14:
   stage: build
-  script: "CI/ci-linux-build.sh gcc C++14 release"
+  script: "CI/ci-linux-build.sh clang cpp14 debug"
   image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
   tags: [Docker]
   artifacts:
     paths:
-    - build-release-gcc-cpp14-Vector-Checks/
+    - build-clang-cpp14-debug-Vector-Checks/
 
-clang-c++14-test-debug:
-  stage: test
-  script: "CI/ci-linux-test.sh clang C++14 debug"
-  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker]
-  dependencies: [clang-c++14-build-debug]
+#-----------  Job Informations:
+# Type:       Test: debug
+# OS:         Linux
+# Compiler:   clang
+# Language:   cpp14
 
-clang-c++14-test-release:
+test-debug-clang-cpp14:
   stage: test
-  script: "CI/ci-linux-test.sh clang C++14 release"
+  script: "CI/ci-linux-test.sh clang cpp14 debug"
   image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
   tags: [Docker]
-  dependencies: [clang-c++14-build-release]
+  dependencies: [build-debug-clang-cpp14]
+
+#-----------  Job Informations:
+# Type:       Build: release
+# OS:         Linux
+# Compiler:   clang
+# Language:   cpp14
 
-clang-c++14-build-debug:
+build-release-clang-cpp14:
   stage: build
-  script: "CI/ci-linux-build.sh clang C++14 debug"
+  script: "CI/ci-linux-build.sh clang cpp14 release"
   image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
   tags: [Docker]
   artifacts:
     paths:
-    - build-debug-clang-cpp14-Vector-Checks/
+    - build-clang-cpp14-release-Vector-Checks/
 
-clang-c++14-build-release:
-  stage: build
-  script: "CI/ci-linux-build.sh clang C++14 release"
+#-----------  Job Informations:
+# Type:       Test: release
+# OS:         Linux
+# Compiler:   clang
+# Language:   cpp14
+
+test-release-clang-cpp14:
+  stage: test
+  script: "CI/ci-linux-test.sh clang cpp14 release"
   image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
   tags: [Docker]
-  artifacts:
-    paths:
-    - build-release-clang-cpp14-Vector-Checks/
+  dependencies: [build-release-clang-cpp14]
+
 
 # -----------------
 # Apple tasks

CI/ci-linux-build.sh

@@ -23,11 +23,11 @@ echo "Building $BUILD_TYPE version with vectorchecks enabled"
 echo "======================================================================"
 echo -e "${NC}"
 
-if [ ! -d build-$BUILD_TYPE_L-$BUILDPATH-Vector-Checks ]; then
-  mkdir build-$BUILD_TYPE_L-$BUILDPATH-Vector-Checks
+if [ ! -d build-$BUILDPATH-Vector-Checks ]; then
+  mkdir build-$BUILDPATH-Vector-Checks
 fi
 
-cd build-$BUILD_TYPE_L-$BUILDPATH-Vector-Checks
+cd build-$BUILDPATH-Vector-Checks
 
 cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DOPENMESH_BUILD_UNIT_TESTS=TRUE -DSTL_VECTOR_CHECKS=ON $OPTIONS ../
 

CI/ci-linux-prepare.sh

@@ -30,14 +30,14 @@ elif [ "$COMPILER" == "clang" ]; then
   BUILDPATH="clang"  
 fi  
 
-if [ "$LANGUAGE" == "C++98" ]; then
+if [ "$LANGUAGE" == "cpp98" ]; then
   echo "Building with C++98";
   BUILDPATH="$BUILDPATH-cpp98"
-elif [ "$LANGUAGE" == "C++11" ]; then
+elif [ "$LANGUAGE" == "cpp11" ]; then
   echo "Building with C++11";
   OPTIONS="$OPTIONS -DCMAKE_CXX_FLAGS='-std=c++11' "
   BUILDPATH="$BUILDPATH-cpp11"  
-elif [ "$LANGUAGE" == "C++14" ]; then
+elif [ "$LANGUAGE" == "cpp14" ]; then
   echo "Building with C++14";
   OPTIONS="$OPTIONS -DCMAKE_CXX_FLAGS='-std=c++14' "
   BUILDPATH="$BUILDPATH-cpp14"  
@@ -51,9 +51,9 @@ OUTPUT='\033[0;32m'
 WARNING='\033[0;93m'
 
 if [ "$BUILD_TYPE" == "release" ]; then
-    export BUILD_TYPE=Release
-    export BUILD_TYPE_L=release
+    export BUILD_TYPE=release
+    BUILDPATH="$BUILDPATH-release"  
 else
-    export BUILD_TYPE=Debug
-    export BUILD_TYPE_L=debug
+    export BUILD_TYPE=debug
+    BUILDPATH="$BUILDPATH-debug"  
 fi
\ No newline at end of file

CI/ci-linux-test.sh

@@ -23,11 +23,11 @@ echo "Building $BUILD_TYPE version unittests"
 echo "======================================================================"
 echo -e "${NC}"
 
-if [ ! -d build-$BUILD_TYPE_L-$BUILDPATH-Vector-Checks ]; then
-  mkdir build-$BUILD_TYPE_L-$BUILDPATH-Vector-Checks
+if [ ! -d build-$BUILDPATH-Vector-Checks ]; then
+  mkdir build-$BUILDPATH-Vector-Checks
 fi
 
-cd build-$BUILD_TYPE_L-$BUILDPATH-Vector-Checks
+cd build-$BUILDPATH-Vector-Checks
 
 #build the unit tests
 make  $MAKE_OPTIONS unittests

CI/gitlab-ci/assemble-gitlab-ci.py

@@ -2,19 +2,29 @@
 
 import sys, os, re
 
+# Version 3
+
 # Script for automated gitlab-ci creation
 # Assembles the gitlab ci from master template file:
 master_file = 'ci-master.yml'
 # Lines in the master file are copied to the resulting
 # assemblied gitlab ci file
 target_file = '../../.gitlab-ci.yml'
-# Lines containing the String {xxx} are interpreted
+# Lines that are {xxx} Strings are interpreted
 # as import statement. Therefore the file xxx is imported
 # into that line.
+# Lines that are {xxx,option1=...,option2=...} includes
+# the file xxx but replaces {{option1}} etc with specified
+# string.
 error_on_path_redirection = True
 # Notice that xxx can not contain path redirections
 # like .. and /
 
+# Max import recursion
+maxFileRecursionDepth = 4
+# Max filename used for pretty print
+maxFilnameChars = 30
+
 
 # Prefix to prepend to master file
 autogenerated_notice = """#############################################################
@@ -25,6 +35,7 @@ autogenerated_notice = """######################################################
 # To make persistent changes changes files in               #
 # ./CI/gitlab-ci/ ...                                       #
 # and regenerate this file with the configuration tool      #
+# python3 ./CI/gitlab-ci/assemble-gitlab-ci.py              #
 #                                                           #
 #############################################################
 
@@ -52,13 +63,37 @@ def readFile(filename):
     file.close()
     return content
 
+# Parse File Import String for variable replacements
+def fetchVariableReplacers(variablesGrep):
+    if (variablesGrep == None):
+        return {}
+
+    regex_option = r"([^\}\n\=,]+)\=([^\}\n\=,]+)"
+    pattern = re.compile(regex_option, flags=re.MULTILINE)
+    result = {}
+
+    for (key, value) in re.findall(pattern, variablesGrep):
+
+        if (key != None and value != None):
+            key = key.strip()
+            result[key] = value
+
+    return result
+
+
 # Assembles the file in memory and returns file content as string
-def assembleTarget(master, depth=3):
+def assembleTarget(master, depth=maxFileRecursionDepth):
     if depth < 0:
         raise "Max depth reached. Possible circular import?"
+    print_prefix = ""
+    for i in range(0, maxFileRecursionDepth-depth):
+        print_prefix = " | \t" + print_prefix
+    print_prefix_inverse = ""
+    for i in range(0, depth):
+        print_prefix_inverse = print_prefix_inverse + "\t"
 
     master_content = readFile(master)
-    regex_import_stmt = r"^\ *\{([^\}\n]+)\}\ *$"
+    regex_import_stmt = r"^\ *\{([^\},\n]+)(,[^=\n\}\,]+\=[^\}\n,]*)*\}\ *$"
     regex_import_comp = re.compile(regex_import_stmt)
     master_content_list = master_content.splitlines()
 
@@ -72,12 +107,21 @@ def assembleTarget(master, depth=3):
             importFile = match.groups()[0]
             if importFile:
                 # Found import statement
-                print("Importing file: "+importFile)
+                print(print_prefix+"Importing file: "+importFile.ljust(maxFilnameChars), end="")
 
                 if not isValidImportFilename(importFile):
                     raise "Invalid filename "+importFile+ ". Do not include path redirections"
+                
+                variablesGrep = match.string
+                variableReplacers = fetchVariableReplacers(variablesGrep)
+
+                print(print_prefix_inverse, variableReplacers)
 
                 import_content = assembleTarget(importFile, depth=depth-1)
+
+                for key, value in variableReplacers.items():
+                    import_content = import_content.replace(r"{{"+key+r"}}", value)
+
                 import_content_list = import_content.splitlines()
                 master_content_list.pop(cur_index)
                 for new_line in reversed(import_content_list):
@@ -108,4 +152,4 @@ def main():
 
 # Execute main function
 if __name__ == '__main__':
-    main()
\ No newline at end of file
+    main()

CI/gitlab-ci/linux-template-build-job.yml

+#-----------  Job Informations:
+# Type:       Build: {{BUILDTYPE}}
+# OS:         Linux
+# Compiler:   {{COMPILER}}
+# Language:   {{LANGUAGE}}
+
+build-{{BUILDTYPE}}-{{COMPILER}}-{{LANGUAGE}}:
+  stage: build
+  script: "CI/ci-linux-build.sh {{COMPILER}} {{LANGUAGE}} {{BUILDTYPE}}"
+  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
+  tags: [Docker]
+  artifacts:
+    paths:
+    - build-{{COMPILER}}-{{LANGUAGE}}-{{BUILDTYPE}}-Vector-Checks/

CI/gitlab-ci/linux-template-job.yml

+{linux-template-build-job.yml, BUILDTYPE=debug}
+
+{linux-template-test-job.yml, BUILDTYPE=debug}
+
+{linux-template-build-job.yml, BUILDTYPE=release}
+
+{linux-template-test-job.yml, BUILDTYPE=release}
+

CI/gitlab-ci/linux-template-test-job.yml

+#-----------  Job Informations:
+# Type:       Test: {{BUILDTYPE}}
+# OS:         Linux
+# Compiler:   {{COMPILER}}
+# Language:   {{LANGUAGE}}
+
+test-{{BUILDTYPE}}-{{COMPILER}}-{{LANGUAGE}}:
+  stage: test
+  script: "CI/ci-linux-test.sh {{COMPILER}} {{LANGUAGE}} {{BUILDTYPE}}"
+  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
+  tags: [Docker]
+  dependencies: [build-{{BUILDTYPE}}-{{COMPILER}}-{{LANGUAGE}}]

CI/gitlab-ci/linux.yml

@@ -6,135 +6,13 @@ cppcheck:
   stage: build
   script: "CI/ci-cppcheck.sh"
   image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker] 
-  artifacts:
-    paths:
-      - cppcheck.log
-
-gcc-c++11-test-debug:
-  stage: test
-  script: "CI/ci-linux-test.sh gcc C++11 debug"
-  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker]
-  dependencies: [gcc-c++11-build-debug]
-
-gcc-c++11-test-release:
-  stage: test
-  script: "CI/ci-linux-test.sh gcc C++11 release"
-  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker]
-  dependencies: [gcc-c++11-build-release]
-
-gcc-c++11-build-debug:
-  stage: build
-  script: "CI/ci-linux-build.sh gcc C++11 debug"
-  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker]
-  artifacts:
-    paths:
-    - build-debug-gcc-cpp11-Vector-Checks/
-
-gcc-c++11-build-release:
-  stage: build
-  script: "CI/ci-linux-build.sh gcc C++11 release"
-  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
   tags: [Docker]
+  timeout: 3h
   artifacts:
     paths:
-    - build-release-gcc-cpp11-Vector-Checks/
-
-clang-c++11-test-debug:
-  stage: test
-  script: "CI/ci-linux-test.sh clang C++11 debug"
-  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker]
-  dependencies: [clang-c++11-build-debug]
-
-clang-c++11-test-release:
-  stage: test
-  script: "CI/ci-linux-test.sh clang C++11 release"
-  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker]
-  dependencies: [clang-c++11-build-release]
-
-clang-c++11-build-debug:
-  stage: build
-  script: "CI/ci-linux-build.sh clang C++11 debug"
-  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker]
-  artifacts:
-    paths:
-    - build-debug-clang-cpp11-Vector-Checks/
-
-clang-c++11-build-release:
-  stage: build
-  script: "CI/ci-linux-build.sh clang C++11 release"
-  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker] 
-  artifacts:
-    paths:
-    - build-release-clang-cpp11-Vector-Checks/
-
-gcc-c++14-test-debug:
-  stage: test
-  script: "CI/ci-linux-test.sh gcc C++14 debug"
-  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker]
-  dependencies: [gcc-c++14-build-debug]
-
-gcc-c++14-test-release:
-  stage: test
-  script: "CI/ci-linux-test.sh gcc C++14 release"
-  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker]
-  dependencies: [gcc-c++14-build-release]
-
-gcc-c++14-build-debug:
-  stage: build
-  script: "CI/ci-linux-build.sh gcc C++14 debug"
-  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker]
-  artifacts:
-    paths:
-    - build-debug-gcc-cpp14-Vector-Checks/
-
-gcc-c++14-build-release:
-  stage: build
-  script: "CI/ci-linux-build.sh gcc C++14 release"
-  image: graphics.rwth-aachen.de:4567/docker/docker/docker-openmesh-container
-  tags: [Docker]
-  artifacts:
-    paths:
-    - build-release-gcc-cpp14-Vector-Checks/
-
-clang-c++14-test-debug:
-  stage: test
-  script: "CI/ci-linux-test.sh clang C++14 debug"