Merge pull request #310 from JCSDA-internal/release/2.0.0

Release/2.0.0

Author: srherbener

.clang-format

@@ -0,0 +1,20 @@
+---
+Language: Cpp
+BasedOnStyle: Google
+AccessModifierOffset: -2
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: true
+AlignEscapedNewlines: Right
+AlignOperands: true
+BreakBeforeBinaryOperators: true
+ColumnLimit: 100
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 2
+FixNamespaceComments: true
+IndentPPDirectives: AfterHash
+IndentWidth: 2
+NamespaceIndentation: None
+PointerAlignment: Right
+ReflowComments: false
+SortIncludes: true
+SortUsingDeclarations: true

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,28 @@
+---
+name: Bug report
+about: Use this template to report bugs
+title: "[Bug]"
+labels: 'bug'
+assignees: ''
+
+---
+
+## Current behavior (describe the bug)
+>[Be sure to add a Pipeline, Label, Estimate, Assignees, and Epic](https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest/inside/practices/issues.html)
+
+## To Reproduce
+
+> What computer are you running on?
+
+> What compilers/modules are you using?
+
+> Steps to reproduce the behavior
+
+1.
+2.
+3.
+...
+
+## Expected behavior
+
+## Additional information (optional)

.github/ISSUE_TEMPLATE/config.yaml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: JCSDA
+    url: https://jcsda.org/
+    about: JCSDA web site
+  - name: Forums
+    url: https://forums.jcsda.org/
+    about: JCSDA user/developer forums

.github/ISSUE_TEMPLATE/general-issue.md

@@ -0,0 +1,26 @@
+---
+name: General issue template
+about: Use this template for general issues
+title: "[New issue]"
+labels: ''
+assignees: ''
+
+---
+
+## Description
+>Provide a detailed description of this issue.
+>What problem needs to be fixed? What new capability needs to be added?
+>If this is a bug, describe the current behavior (or use the bug template).
+>[Be sure to add a Pipeline, Label, Estimate, Assignees, and Epic](https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest/inside/practices/issues.html)
+
+## Requirements
+
+>If this is a new feature: What does the new code need to accomplish? Does it require new software dependencies (e.g. new jedi-stack components or new python modules?)
+>If this is a bugfix: What is the expected behavior?
+
+## Acceptance Criteria (Definition of Done)
+>What does it mean for this to be finished?
+
+## Dependencies
+>What must be done before this can be done? Add issue dependencies in ZenHub as appropriate
+>Does this block progress on other issues? Add this issue as a dependency to other ZenHub issues as appropriate

.gitignore

@@ -1,3 +1,10 @@
 *DS_Store*
 ._*
 *.swp
+/.vs
+/out/build
+build
+/ioda-data
+/jedicmake
+/CMakeSettings.json
+.vs

CI/CMakeLists.txt

@@ -1,4 +1,4 @@
-# (C) Copyright 2017 UCAR
+# (C) Copyright 2017-2021 UCAR
 #
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
@@ -9,22 +9,52 @@
 
 cmake_minimum_required( VERSION 3.12 FATAL_ERROR )
 
-project( ioda-bundle LANGUAGES C CXX Fortran )
+project( ioda-bundle VERSION 1.0.0 LANGUAGES C CXX Fortran )
+
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 
-find_package(ecbuild)
-set( CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake;${CMAKE_CURRENT_SOURCE_DIR}/ecbuild/cmake;${CMAKE_MODULE_PATH}")
 include( ecbuild_bundle )
 
 set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" )
 
 ecbuild_bundle_initialize()
 
-option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON") #Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF
-ecbuild_bundle( PROJECT fckit GIT "https://github.com/JCSDA-internal/fckit.git" )
-ecbuild_bundle( PROJECT atlas GIT "https://github.com/JCSDA-internal/atlas.git" )
+ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/JCSDA-internal/jedi-cmake.git" )
+include( jedicmake/cmake/Functions/git_functions.cmake )
+
+# ECMWF libs
+# ----------
+option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF
+option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Build fckit unless user passes -DBUNDLE_SKIP_FCKIT=OFF
+option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Build atlas unless user passes -DBUNDLE_SKIP_ATLAS=OFF
+
+ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 )
+ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 )
+ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.24.1 )
+
+# Core JEDI repositories
+ecbuild_bundle( PROJECT oops GIT "https://github.com/JCSDA-internal/oops.git" )
+ecbuild_bundle( PROJECT ioda GIT "https://github.com/JCSDA-internal/ioda.git" )
+
+
+# If IODA branch is being built set GIT_BRANCH_FUNC to IODA's current branch.
+# If a tagged version of IODA is being built set GIT_TAG_FUNC to ioda's current tag. In this case,
+# IODA test files will be download from UCAR DASH and ioda-data repo will not be cloned.
+find_branch_name(REPO_DIR_NAME ioda)
+
+# When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored
+# in a local directory, ioda-data repo will not be cloned
+if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} )
+  ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" )
+endif()
+
+# If IODA's current branch is available in ioda-data repo, that branch will be checked out
+branch_checkout (REPO_DIR_NAME ioda-data
+                 BRANCH ${GIT_BRANCH_FUNC} )
+
 
-ecbuild_bundle( PROJECT oops         GIT "https://github.com/JCSDA/oops.git" )
-ecbuild_bundle( PROJECT ioda         GIT "https://github.com/JCSDA/ioda.git" )
+# Build IODA converters
+#ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" )
 
 # Build Doxygen documentation
 option(BUILD_IODA_BUNDLE_DOC "Build documentation" OFF)

CI/buildspec_clang.yml

@@ -1,5 +1,5 @@
 version: 0.2
-  
+
 env:
   parameter-store:
     GIT_USER: "/CodeBuild/Git_USER"
@@ -22,8 +22,11 @@ phases:
       - echo $CODEBUILD_WEBHOOK_TRIGGER
       - echo $CODEBUILD_WEBHOOK_BASE_REF
 
-      - ls
-      - if [ "$CODEBUILD_WEBHOOK_EVENT" = "PULL_REQUEST_MERGED" ];
+      # Codebuild only runs on PUSH events if HEAD_REF
+      # is refs/heads/develop (merge to develop). In this
+      # case CODEBUILD_GIT_BRANCH="develop"
+
+      - if [ "$CODEBUILD_WEBHOOK_EVENT" = "PUSH" ];
         then export CODEBUILD_GIT_BRANCH="develop";
         echo "Merging to develop";
         else export CODEBUILD_GIT_BRANCH=${CODEBUILD_WEBHOOK_HEAD_REF#refs/heads/};
@@ -37,13 +40,102 @@ phases:
       - echo Executing pre_build phase
       - git lfs install # creates .gitconfig
       - cp ~/.gitconfig .
+      - mkdir /build_container
+      - mkdir /jcsda
+
+      - if [ "$CODEBUILD_GIT_BRANCH" = "develop" ];
+        then export CODEBUILD_GIT_BRANCH_FORK="release-stable";
+        export CODEBUILD_GIT_BRANCH_CRTM=${CRTM_JEDI_BRANCH};
+        else export CODEBUILD_GIT_BRANCH_FORK=${CODEBUILD_GIT_BRANCH};
+        export CODEBUILD_GIT_BRANCH_CRTM=${CODEBUILD_GIT_BRANCH};
+        echo "CODEBUILD_GIT_BRANCH_FORK=${CODEBUILD_GIT_BRANCH_FORK}";
+        fi
+
+      # Upload branch name and commit sha as CodeBuild artifact to S3
+      - mkdir -p /jcsda/artifacts
+      - echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt
+      - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.tx
+
+      - cd CI
+
+      # jedi-cmake
+      - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/jedi-cmake $CODEBUILD_GIT_BRANCH jedicmake /jcsda/ioda-bundle develop
+
+      # ioda
+      - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/ioda $CODEBUILD_GIT_BRANCH ioda /jcsda/ioda-bundle develop
+
+      # oops
+      - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/oops $CODEBUILD_GIT_BRANCH oops /jcsda/ioda-bundle develop
+
+      # internal repos
+      #fckit
+      - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/fckit $CODEBUILD_GIT_BRANCH_FORK fckit /jcsda/ioda-bundle release-stable
+
+      # atlas
+      - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/atlas $CODEBUILD_GIT_BRANCH_FORK atlas /jcsda/ioda-bundle release-stable
 
+      # ioda-test
+      - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/ioda-data $CODEBUILD_GIT_BRANCH ioda-data /jcsda/ioda-bundle develop
+
+      - cp CMakeLists.txt /jcsda/ioda-bundle
+      - cp -r cmake /jcsda/ioda-bundle/
+      # cdash upload setup
+      - cp /jcsda/ioda-bundle/cmake/CTestConfig.cmake /jcsda/ioda-bundle
+ 
+      - chmod 777 -R /jcsda/ioda-bundle
+      - cd /jcsda/ioda-bundle
+      - ls
 
-      # get jedi-build-package
-      - git clone https://$GIT_USER:[email protected]/jcsda-internal/jedi-build-package
-      - cd jedi-build-package
-      - pip install --user -e .
   build:
+    on-failure: CONTINUE
     commands:
-      - ~/.local/bin/jedi-build --gh-token=$GIT_PASS -j 4 --env-id=docker-clang --branch-map atlas:release-stable fckit:release-stable -br ${CODEBUILD_GIT_BRANCH} -p ioda -vvv -3 -u --submit-dashboard --site AWS --abort-on-build-errors --abort-on-test-errors
+      - echo Executing build phase
+      - echo $CODEBUILD_BUILD_SUCCEEDING
+      - export BUILD_STATUS="0"
+      - echo $BUILD_STATUS
+
+      # configure and build
+      - cd /build_container
+      - ecbuild -Wno-dev -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE -DBUILD_PYTHON_BINDINGS=ON /jcsda/ioda-bundle
+      - cd /build_container/ioda
+      - cp ../DartConfiguration.tcl .
+      - sed -i 's/ioda-bundle/ioda-bundle\/ioda/' DartConfiguration.tcl
+      - sed -i 's/build_container/build_container\/ioda/' DartConfiguration.tcl
+      - cat DartConfiguration.tcl
+      - make -j4
+
+      - if [ "$CODEBUILD_BUILD_SUCCEEDING" = "1" ];
+        then export BUILD_STATUS="1";
+        echo "Build passed";
+        fi
+      - echo $BUILD_STATUS
+
+      # run ctest
+      - cd /build_container/ioda
+      - ctest -C RelWithDebInfo -D ExperimentalTest
+
+    finally:
+      - cd /build_container/ioda
+      - ctest -C RelWithDebInfo -D ExperimentalSubmit -M Continuous -- --track Continuous --group Continuous
+
+  post_build:
+    commands:
+      - echo Executing post_build phase
+      - echo $CODEBUILD_BUILD_SUCCEEDING
+      - echo $BUILD_STATUS
+
+      # upload find cdash url and upload it as CodeBuild artifact to S3
+      - if [ "$BUILD_STATUS" = "1" ];
+        then echo "Build & tests passed, find cdash url";
+        bash /jcsda/ioda-bundle/ioda/CI/cdash-url.sh /build_container/ioda/Testing;
+        url=$(bash /jcsda/ioda-bundle/ioda/CI/cdash-url.sh /build_container/ioda/Testing);
+        echo $url;
+        echo ${url} > /jcsda/artifacts/cdash-url.txt;
+        cat /jcsda/artifacts/cdash-url.txt;
+        else echo "Build failed";
+        fi
 
+artifacts:
+  files:
+    - '/jcsda/artifacts/*'
+  name: ioda-clang-url