esp/ci: Move newlib build to separate job

Author: gerekon

.gitlab-ci.yml

@@ -1,5 +1,6 @@
 stages:
   - build
+  - pack
   - private_deploy
   - test
   - public_deploy
@@ -19,14 +20,16 @@ variables:
   BINUTILS_REF: "esp-2022r1-binutils"
   XTENSA_OVERLAYS_REPO: "xtensa-overlays"
   XTENSA_OVERLAYS_REF: "master"
-  # TODO: update vars below to tags names after related branches are merged in those repos
+  LLVM_GCC_TESTSUITE_REF: "release_universal_clang_toolchain"
+  # TODO: update var below to tags names after related branches are merged in those repos
+  # XTENSA_CLANG_TOOLCHAIN_REF: "release_universal_clang_toolchain"
   XTENSA_CLANG_TOOLCHAIN_REF: "build_macos_arm64"
-  LLVM_GCC_TESTSUITE_REF: "feature/toolchain_build_script"
 
   PLATFORM_NAME_LINUX_ARM64: "linux-arm64"
   PLATFORM_NAME_LINUX: "linux-amd64"
   PLATFORM_NAME_WIN: "win64"
   PLATFORM_NAME_MACOS: "macos"
+  PLATFORM_NAME_MACOS_ARM64: "macos-arm64"
 
   ARCHIVE_TOOL_LINUX: "tar -cJf"
   UNARCHIVE_TOOL_LINUX: "tar -xf"
@@ -40,6 +43,11 @@ variables:
   UNARCHIVE_TOOL_MACOS: "tar -xf"
   ARCHIVE_EXT_MACOS: "tar.xz"
 
+  ARCHIVE_TOOL_NEWLIB: ${ARCHIVE_TOOL_LINUX}
+  UNARCHIVE_TOOL_NEWLIB: ${UNARCHIVE_TOOL_LINUX}
+  ARCHIVE_EXT_NEWLIB: ${ARCHIVE_EXT_LINUX}
+
+  DIST_NEW_DIR: "_dist_new"
   DIST_DIR: "dist"
   BUILD_DIR: "_build"
   DOWNLOADS_DIR: "_downloads"
@@ -110,7 +118,19 @@ before_script:
   ${ARCHIVE_TOOL} ${LIBS_ARCHIVE_NAME} esp-clang/lib/libclang* esp-clang/lib/clang/${CLANG_VER}/include
   mkdir -p ${DISTRO_DIR}
   mv ${LIBS_ARCHIVE_NAME} ${DISTRO_DIR}/
-  echo "${LIBS_ARCHIVE_NAME}" > ${DISTRO_DIR}/file_lib${PLATFORM_NAME}
+  echo "${LIBS_ARCHIVE_NAME}" > ${DISTRO_DIR}/file_libs-${PLATFORM_NAME}
+
+.get_binutils: &get_binutils |
+  git clone -b ${BINUTILS_REF} --single-branch ${GITLAB_SSH_SERVER}/idf/${BINUTILS_REPO}.git
+  BINUTILS_PATH=$PWD/${BINUTILS_REPO}
+
+.get_xtensa_overlays: &get_xtensa_overlays |
+  git clone -b ${XTENSA_OVERLAYS_REF} --single-branch ${GITLAB_SSH_SERVER}/idf/${XTENSA_OVERLAYS_REPO}.git
+  XTENSA_OVERLAYS_PATH=$PWD/${XTENSA_OVERLAYS_REPO}
+
+.get_newlib: &get_newlib |
+  git clone -b ${NEWLIB_REF} --single-branch ${GITLAB_SSH_SERVER}/idf/${NEWLIB_REPO}.git
+  NEWLIB_PATH=$PWD/${NEWLIB_REPO}
 
 .build_template:
   stage: build
@@ -119,77 +139,49 @@ before_script:
   artifacts:
     paths:
       - ${DIST_DIR}/
-      - newlib/
-      - ${BUILD_DIR}/clang_tests.log
-      - ${BUILD_DIR}/clang_build.log
-      - ${BUILD_DIR}/newlib_build.log
+      - ${BUILD_DIR}/tests.log
+      - ${BUILD_DIR}/build.log
     when: always
-    expire_in: 3 day
+    expire_in: 1 day
   variables:
-    BUILD_TOOLCHAIN_CMD_ARGS: ""
+    BUILD_TOOLCHAIN_CMD_EXTRA_ARGS: ""
     # use separate dist dir for universal toolchain
     # TODO: remove this var after switching to universal toolchain builds
-    DIST_DIR: "dist_new"
+    DIST_DIR: ${DIST_NEW_DIR}
   script:
     - *get_release_name
     - mkdir ${DOWNLOADS_DIR}
     - pushd ${DOWNLOADS_DIR}
-    - export ESP_GCC_TOOLCHAIN_DIST_BASE=$PWD
+    - ESP_GCC_TOOLCHAIN_DIST_BASE=$PWD
     - *get_gcc_toolchain
-    - git clone -b ${BINUTILS_REF} --single-branch ${GITLAB_SSH_SERVER}/idf/${BINUTILS_REPO}.git
-    - export BINUTILS_PATH=$PWD/${BINUTILS_REPO}
-    - git clone -b ${XTENSA_OVERLAYS_REF} --single-branch ${GITLAB_SSH_SERVER}/idf/${XTENSA_OVERLAYS_REPO}.git
-    - export XTENSA_OVERLAYS_PATH=$PWD/${XTENSA_OVERLAYS_REPO}
+    - *get_binutils
+    - *get_xtensa_overlays
     - popd
     - *get_clang_toolchain_build_scripts
     - *fix_origin_remote_for_public
-    - export ESP_GCC_TOOLCHAIN_REL_VER=${GCC_REL_NAME}
-    - export LLVM_PROJECT_PATH=$PWD
-    - export BUILD_PATH=$PWD/${BUILD_DIR}
+    - LLVM_PROJECT_PATH=$PWD
+    - BUILD_PATH=$PWD/${BUILD_DIR}
     - mkdir -p ${BUILD_PATH}
     - export USE_PARALLEL_LINK_JOBS=2
     # build Clang toolchain w/o newlib
     - ${BUILD_TOOLCHAIN_CMD} --llvm-path=${LLVM_PROJECT_PATH}
       --gcc-toolchains-path=${ESP_GCC_TOOLCHAIN_DIST_BASE} --binutils-path=${BINUTILS_PATH}
-      --xtensa-overlays-path=${XTENSA_OVERLAYS_PATH} --host=${CONF_HOST} ${BUILD_TOOLCHAIN_CMD_ARGS} ${BUILD_PATH} 2>&1 > ${BUILD_PATH}/clang_build.log
-    # use just built Clang to build newlib
-    - export PATH=${BUILD_PATH}/esp-clang/bin:$PATH
-    - export BUILD_HOST=$(gcc -dumpmachine)
-    - export NEWLIB_OVERLAY_DISTRO_PATH=$PWD/newlib;
-    # build newlib overlay using ESP native (Linux) clang toolchain only
-    # it will be re-used for cross-buit toolchains (win and mac).
-    # FIXME: it would be good to move newlib overlay build to separate job and have job sequence like
-    # clang_linux_wo_newlib -> newlib_overlay -> clang_linux_full(copy newlib) -> clang_linux_unittests
-    # but we need full native (Linux) toolchain to run unittests and unittests need clang build dir.
-    # clang build dir may occupy about 2GB, so it looks too heavy to pass it as artifact
-    - if [ "${CONF_HOST}" == "${BUILD_HOST}" ]; then
-        export BUILD_NEWLIB_PATH=${BUILD_PATH}/newlib;
-        mkdir -p ${NEWLIB_OVERLAY_DISTRO_PATH};
-        git clone -b ${NEWLIB_REF} --single-branch ${GITLAB_SSH_SERVER}/idf/${NEWLIB_REPO}.git;
-        export NEWLIB_PATH=$PWD/${NEWLIB_REPO};
-        ./build-toolchain.sh --newlib-path=${NEWLIB_PATH} --xtensa-overlays-path=${XTENSA_OVERLAYS_PATH} ${BUILD_NEWLIB_PATH} 2>&1 > ${BUILD_PATH}/newlib_build.log;
-        pushd ${BUILD_NEWLIB_PATH};
-        ${ARCHIVE_TOOL_LINUX} ${NEWLIB_OVERLAY_DISTRO_PATH}/esp-clang-newlib-overlay.${ARCHIVE_EXT_LINUX} esp-clang/;
-        popd;
-      fi
-    - ${UNARCHIVE_TOOL_LINUX} ${NEWLIB_OVERLAY_DISTRO_PATH}/esp-clang-newlib-overlay.${ARCHIVE_EXT_LINUX} -C ${BUILD_PATH}
-    # strip binutils afer newlib is built
-    - STRIP_BINUTILS=YES ./build-toolchain.sh --host=${CONF_HOST} ${BUILD_PATH}
-    # Run unit tests for native build only.
+      --xtensa-overlays-path=${XTENSA_OVERLAYS_PATH} --host=${CONF_HOST} ${BUILD_TOOLCHAIN_CMD_EXTRA_ARGS} ${BUILD_PATH} 2>&1 > ${BUILD_PATH}/build.log
+    - BUILD_HOST=$(gcc -dumpmachine)
+    # Do not run unit tests for cross-builds.
     # Run as non-root user because permission tests fail when run by root.
     - if [ "${CONF_HOST}" == "${BUILD_HOST}" ]; then
         export LLVM_BUILD_PATH=${LLVM_PROJECT_PATH}/llvm/build-Release-${CONF_HOST};
-        echo "Run unit tests for native build ib ${LLVM_BUILD_PATH}";
+        echo "Run unit tests for native build in ${LLVM_BUILD_PATH}";
         useradd -m test_runner;
         chown -R test_runner ${LLVM_BUILD_PATH};
-        touch ${BUILD_PATH}/clang_tests.log;
-        chmod o+w ${BUILD_PATH}/clang_tests.log;
-        runuser -l test_runner -c 'cmake --build '${LLVM_BUILD_PATH}' --target check-all 2>&1 > '${BUILD_PATH}'/clang_tests.log';
+        touch ${BUILD_PATH}/tests.log;
+        chmod o+w ${BUILD_PATH}/tests.log;
+        runuser -l test_runner -c 'cmake --build '${LLVM_BUILD_PATH}' --target check-all 2>&1 > '${BUILD_PATH}'/tests.log';
       fi
     - export DISTRO_DIR=$PWD/$DIST_DIR
     - pushd ${BUILD_PATH}
     - *package_toolchain
-    - *package_libs
     - popd
 
 build_x86_64-linux-gnu:
@@ -202,75 +194,195 @@ build_x86_64-linux-gnu:
     ARCHIVE_EXT: "${ARCHIVE_EXT_LINUX}"
     BUILD_TOOLCHAIN_CMD: "./build-toolchain.sh"
 
-.build_x86_64-w64-mingw32:
+build_x86_64-w64-mingw32:
   extends: .build_template
   needs:
+    # needs native toolchain and newlib from this job
     - job: build_x86_64-linux-gnu
   before_script:
     - *use_ci_tools
     - *add_gitlab_key
     # get ARCHIVE_NAME for Linux release. Modify vars to make get_release_name working properly
-    - export PLATFORM_NAME_ORIG=${PLATFORM_NAME}
-    - export ARCHIVE_EXT_ORIG=${ARCHIVE_EXT}
-    - export PLATFORM_NAME=${PLATFORM_NAME_LINUX}
-    - export ARCHIVE_EXT=${ARCHIVE_EXT_LINUX}
-    - *get_release_name
-    # restore modified vars 
-    - export PLATFORM_NAME=${PLATFORM_NAME_ORIG}
-    - export ARCHIVE_EXT=${ARCHIVE_EXT_ORIG}
-    # unpack Linux release to re-use it as native Clang for Windows build
+    - CLANG_LINUX_ARCHIVE=$(cat ${DIST_DIR}/file_${PLATFORM_NAME_LINUX})
+    # unpack x86_64-linux-gnu toolchain to re-use it as native Clang for Windows build
     - mkdir -p esp-clang-${PLATFORM_NAME_LINUX}
-    - ${UNARCHIVE_TOOL_LINUX} ${DIST_DIR}/${ARCHIVE_NAME} -C esp-clang-${PLATFORM_NAME_LINUX}
+    - ${UNARCHIVE_TOOL_LINUX} ${DIST_DIR}/${CLANG_LINUX_ARCHIVE} -C esp-clang-${PLATFORM_NAME_LINUX}
     # we do not want to keep artifacts from 'x86_64-linux-gnu' job
     - rm -rf ${DIST_DIR}
     - rm -rf ${BUILD_DIR}
     # add build command args speciifc for Windows build
-    - export BUILD_TOOLCHAIN_CMD_ARGS="--native-esp-clang-path=$PWD/esp-clang-${PLATFORM_NAME_LINUX}"
+    - export BUILD_TOOLCHAIN_CMD_EXTRA_ARGS="--native-esp-clang-path=$PWD/esp-clang-${PLATFORM_NAME_LINUX}"
   variables:
     CONF_HOST: "x86_64-w64-mingw32"
     PLATFORM_NAME: "${PLATFORM_NAME_WIN}"
-    ARCHIVE_TOOL: "${ARCHIVE_TOOL_WIN}"
+    # Use Linux compressor to minimize artifact size.
+    # Toolchain is not fully stripped yet, so may exceed max artifact size.
+    ARCHIVE_TOOL: "${ARCHIVE_TOOL_LINUX}"
     UNARCHIVE_TOOL: "${UNARCHIVE_TOOL_WIN}"
     ARCHIVE_EXT: "${ARCHIVE_EXT_WIN}"
     BUILD_TOOLCHAIN_CMD: "./build-toolchain-win.sh"
 
 .build_apple-darwin_template:
   extends: .build_template
-  needs:
-    - job: build_x86_64-linux-gnu
   variables:
-    PLATFORM_NAME: "${PLATFORM_NAME_MACOS}"
     ARCHIVE_TOOL: "${ARCHIVE_TOOL_MACOS}"
     UNARCHIVE_TOOL: "${UNARCHIVE_TOOL_MACOS}"
     ARCHIVE_EXT: "${ARCHIVE_EXT_MACOS}"
-    GCC_ARCHIVE_EXT: "${GCC_ARCHIVE_EXT_MACOS}"
 
 build_x86_64-apple-darwin:
   extends: .build_apple-darwin_template
   variables:
     CONF_HOST: "x86_64-apple-darwin21.1"
+    PLATFORM_NAME: "${PLATFORM_NAME_MACOS}"
     BUILD_TOOLCHAIN_CMD: "./build-toolchain-macos.sh --host-arch=x86_64"
 
 build_aarch64-apple-darwin:
   extends: .build_apple-darwin_template
   variables:
     CONF_HOST: "aarch64-apple-darwin21.1"
+    PLATFORM_NAME: "${PLATFORM_NAME_MACOS_ARM64}"
     BUILD_TOOLCHAIN_CMD: "./build-toolchain-macos.sh --host-arch=aarch64"
 
-.test_x86_64-linux-gnu:
-  stage: test
+build_newlib:
+  stage: build
+  tags: [ "amd64", "build" ]
+  needs:
+    # needs native toolchainfrom this job
+    - job: build_x86_64-linux-gnu
+  artifacts:
+    paths:
+      - ${DIST_DIR}/
+      - ${BUILD_DIR}/build.log
+    when: always
+    expire_in: 1 day
+  variables:
+    PLATFORM_NAME: "${PLATFORM_NAME_LINUX}"
+    ARCHIVE_TOOL: "${ARCHIVE_TOOL_LINUX}"
+    UNARCHIVE_TOOL: "${UNARCHIVE_TOOL_LINUX}"
+    ARCHIVE_EXT: "${ARCHIVE_EXT_LINUX}"
+    # use separate dist dir for universal toolchain
+    # TODO: remove this var after switching to universal toolchain builds
+    DIST_DIR: ${DIST_NEW_DIR}
+  script:
+    # get ARCHIVE_NAME for Linux release.
+    - CLANG_ARCHIVE=$PWD/${DIST_DIR}/$(cat ${DIST_DIR}/file_${PLATFORM_NAME_LINUX})
+    - mkdir -p ${DOWNLOADS_DIR}
+    - pushd ${DOWNLOADS_DIR}
+    - *get_xtensa_overlays
+    - *get_newlib
+    # unpack clang
+    - ${UNARCHIVE_TOOL} ${CLANG_ARCHIVE}
+    - export PATH=$PWD/esp-clang/bin:$PATH
+    - popd
+    - rm -rf $PWD/${DIST_DIR}
+    - *get_clang_toolchain_build_scripts
+    # build newlib overlay using ESP native (Linux) clang toolchain only
+    # it will be re-used for cross-buit toolchains (win and mac).
+    - NEWLIB_OVERLAY_DISTRO_PATH=$PWD/${DIST_DIR}
+    - mkdir -p ${NEWLIB_OVERLAY_DISTRO_PATH}
+    - BUILD_PATH=$PWD/${BUILD_DIR}
+    - mkdir -p ${BUILD_PATH}
+    - ./build-toolchain.sh --newlib-path=${NEWLIB_PATH} --xtensa-overlays-path=${XTENSA_OVERLAYS_PATH} ${BUILD_PATH} 2>&1 > ${BUILD_PATH}/build.log
+    - pushd ${BUILD_PATH}
+    - ${ARCHIVE_TOOL_NEWLIB} ${NEWLIB_OVERLAY_DISTRO_PATH}/esp-clang-newlib-overlay.${ARCHIVE_EXT_NEWLIB} esp-clang/
+    - popd
+
+.pack_template:
+  stage: pack
   tags: [ "amd64", "build" ]
   allow_failure: true
+  artifacts:
+    paths:
+      - ${DIST_DIR}/
+    when: always
+    expire_in: 3 day
+  variables:
+    # use separate dist dir for universal toolchain
+    # TODO: remove this var after switching to universal toolchain builds
+    DIST_DIR: ${DIST_NEW_DIR}
+  script:
+    - *get_release_name
+    - export BUILD_PATH=$PWD/${BUILD_DIR}
+    - mkdir -p ${BUILD_PATH}
+    # unpack clang
+    - ${UNARCHIVE_TOOL} ${DIST_DIR}/${ARCHIVE_NAME} -C ${BUILD_PATH}
+    # unpack newlib
+    - ${UNARCHIVE_TOOL_NEWLIB} ${DIST_DIR}/esp-clang-newlib-overlay.${ARCHIVE_EXT_NEWLIB} -C ${BUILD_PATH}
+    - rm -rf ${DIST_DIR}
+    - *get_clang_toolchain_build_scripts
+    # strip binutils afer newlib is built
+    - STRIP_BINUTILS=YES ./build-toolchain.sh --host=${CONF_HOST} ${BUILD_PATH}
+    - DISTRO_DIR=$PWD/${DIST_DIR}
+    - pushd ${BUILD_PATH}
+    - *package_toolchain
+    - *package_libs
+    - popd
+
+pack_x86_64-linux-gnu:
+  extends: .pack_template
   needs:
     - job: build_x86_64-linux-gnu
+    - job: build_newlib
+  variables:
+    CONF_HOST: "x86_64-linux-gnu"
+    PLATFORM_NAME: "${PLATFORM_NAME_LINUX}"
+    ARCHIVE_TOOL: "${ARCHIVE_TOOL_LINUX}"
+    UNARCHIVE_TOOL: "${UNARCHIVE_TOOL_LINUX}"
+    ARCHIVE_EXT: "${ARCHIVE_EXT_LINUX}"
+
+pack_x86_64-w64-mingw32:
+  extends: .pack_template
+  needs:
+    - job: build_x86_64-w64-mingw32
+    - job: build_newlib
+  variables:
+    CONF_HOST: "x86_64-w64-mingw32"
+    PLATFORM_NAME: "${PLATFORM_NAME_WIN}"
+    # use Linux compressor to save space.
+    # upon release archive will be re-packed into zip format for uploading to GH
+    ARCHIVE_TOOL: "${ARCHIVE_TOOL_LINUX}"
+    UNARCHIVE_TOOL: "${UNARCHIVE_TOOL_LINUX}"
+    ARCHIVE_EXT: "${ARCHIVE_EXT_WIN}"
+
+.pack_apple-darwin_template:
+  extends: .pack_template
+  variables:
+    ARCHIVE_TOOL: "${ARCHIVE_TOOL_MACOS}"
+    UNARCHIVE_TOOL: "${UNARCHIVE_TOOL_MACOS}"
+    ARCHIVE_EXT: "${ARCHIVE_EXT_MACOS}"
+
+pack_x86_64-apple-darwin:
+  extends: .pack_apple-darwin_template
+  needs:
+    - job: build_x86_64-apple-darwin
+    - job: build_newlib
+  variables:
+    CONF_HOST: "x86_64-apple-darwin21.1"
+    PLATFORM_NAME: "${PLATFORM_NAME_MACOS}"
+
+pack_aarch64-apple-darwin:
+  extends: .pack_apple-darwin_template
+  needs:
+    - job: build_aarch64-apple-darwin
+    - job: build_newlib
+  variables:
+    CONF_HOST: "aarch64-apple-darwin21.1"
+    PLATFORM_NAME: "${PLATFORM_NAME_MACOS_ARM64}"
+
+test_x86_64-linux-gnu:
+  stage: test
+  tags: [ "amd64", "build" ]
+  allow_failure: true
+  needs:
+    - job: pack_x86_64-linux-gnu
   variables:
     PLATFORM_NAME: "${PLATFORM_NAME_LINUX}"
     ARCHIVE_TOOL: "${ARCHIVE_TOOL_LINUX}"
     UNARCHIVE_TOOL: "${UNARCHIVE_TOOL_LINUX}"
     ARCHIVE_EXT: "${ARCHIVE_EXT_LINUX}"
     # use separate dist dir for universal toolchain
     # TODO: remove this var after switching to universal toolchain builds
-    DIST_DIR: "dist_new"
+    DIST_DIR: ${DIST_NEW_DIR}
   script:
     - *get_release_name
     - ${UNARCHIVE_TOOL} ${DIST_DIR}/${ARCHIVE_NAME}