From 66471c3133575fdac05daea8c2b0ea5a98c32645 Mon Sep 17 00:00:00 2001 From: Nizamudeen A Date: Mon, 27 Jun 2022 12:14:47 +0530 Subject: [PATCH] improve multicluster support Adapting the script to run with the mstart. I introduced a new variable called `NUMBER_OF_CLUSTERS`. All the clusters deployed are on the same container but the dashboard is accessbile on different ports. The first port is configurable and a 100 will be added to the ports of next cluster. for eg. CEPH1 is in 11000 and CEPH2 is in 11100 and CEPH3 is in 11200.... LIMITATIONS SO FAR DEV SERVER ON FIRST CLUSTER ONLY localhost:11000 is somehow not there :( MONITORING STACKS CAN ONLY DEPLOYED ON FIRST CLUSTER Signed-off-by: Nizamudeen A --- .env.example | 1 + docker-compose.yml | 1 + docker/ceph/ci/sanity-checks.sh | 9 ++++++-- docker/ceph/entrypoint.sh | 15 ++++++------ docker/ceph/set-monitoring.sh | 11 ++++----- docker/ceph/set-mstart-env.sh | 39 +++++++++++++++++++++++++++++++ docker/ceph/set-start-env.sh | 2 ++ docker/ceph/start-ceph.sh | 41 ++++++++++++++++++++++----------- 8 files changed, 91 insertions(+), 28 deletions(-) create mode 100755 docker/ceph/set-mstart-env.sh diff --git a/.env.example b/.env.example index cbf4a0e504c..cdce5555294 100644 --- a/.env.example +++ b/.env.example @@ -41,6 +41,7 @@ DASHBOARD_DEV_SERVER=1 DASHBOARD_SSL=1 DASHBOARD_URL= RGW_MULTISITE=0 +NUMBER_OF_CLUSTERS=1 # Sanity checks' options #APPLITOOLS_API_KEY= diff --git a/docker-compose.yml b/docker-compose.yml index 8c6ec1c358d..d35bd1ff96d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,7 @@ services: - PYTHONDONTWRITEBYTECODE=1 - RGW - RGW_MULTISITE=${RGW_MULTISITE:-0} + - NUMBER_OF_CLUSTERS=${NUMBER_OF_CLUSTERS:-1} cap_add: - ALL entrypoint: /docker/entrypoint.sh diff --git a/docker/ceph/ci/sanity-checks.sh b/docker/ceph/ci/sanity-checks.sh index e4422090dd1..8fa76202388 100755 --- a/docker/ceph/ci/sanity-checks.sh +++ b/docker/ceph/ci/sanity-checks.sh @@ -2,6 +2,8 @@ set -e +source /docker/set-mstart-env.sh + REPO_DIR=/ceph [[ "$IS_UPSTREAM" == 1 && "$CEPH_VERSION" -le '14' ]] && PYTHON_VERSION=2 || PYTHON_VERSION=3 TRANSLATION_FILE=src/pybind/mgr/dashboard/frontend/src/locale/messages.xlf @@ -244,7 +246,7 @@ run_frontend_e2e_tests() { echo 'Running frontend E2E tests...' cd "$REPO_DIR"/src/pybind/mgr/dashboard/frontend - npm ci + npm i npx --no-install cypress -v && WITH_CYPRESS=1 || WITH_CYPRESS=0 E2E_CMD="npm run e2e:dev" if [[ "${WITH_CYPRESS}" == 1 ]]; then @@ -264,12 +266,15 @@ run_frontend_e2e_tests() { DASHBOARD_URL=null while [[ "${DASHBOARD_URL}" == 'null' ]]; do - export DASHBOARD_URL=$("$CEPH_BIN"/ceph mgr services | jq -r .dashboard) + export DASHBOARD_URL=$(CEPH_CLI mgr services | jq -r .dashboard) + export DASHBOARD2_URL=$(CEPH2_CLI mgr services | jq -r .dashboard) sleep 1 done if [[ "${WITH_CYPRESS}" == 1 ]]; then export CYPRESS_BASE_URL="${DASHBOARD_URL}" + export CYPRESS_CEPH2_URL="${DASHBOARD2_URL}" fi + cd "$REPO_DIR"/src/pybind/mgr/dashboard/frontend ANGULAR_VERSION=$(npm run ng version | grep 'Angular: ' | awk '{ print substr($2,1,1) }') # In nautilus this flag is required because BASE_URL is not read in protractor config. diff --git a/docker/ceph/entrypoint.sh b/docker/ceph/entrypoint.sh index 36d61c66342..dbb36cd9cdf 100755 --- a/docker/ceph/entrypoint.sh +++ b/docker/ceph/entrypoint.sh @@ -10,22 +10,23 @@ export_var() { export_var IS_CEPH_RPM="$(command -v ceph-mgr 2>/dev/null | wc -l)" # Env. vars used in vstart. -export_var CEPH_ASOK_DIR=/ceph/build."${HOSTNAME}"/out -export_var CEPH_CONF=/ceph/build."${HOSTNAME}"/ceph.conf -export_var CEPH_CONF_PATH=/ceph/build."${HOSTNAME}" -export_var CEPH_DEV_DIR=/ceph/build."${HOSTNAME}"/dev -export_var CEPH_OUT_DIR=/ceph/build."${HOSTNAME}"/out +export_var CEPH_ASOK_DIR=/ceph/build.ceph-dev/run/"${HOSTNAME}"/asok +export_var CEPH_CONF=/ceph/build.ceph-dev/run/"${HOSTNAME}"/ceph.conf +export_var CEPH_CONF_PATH=/ceph/build.ceph-dev/run/"${HOSTNAME}" +export_var CEPH_DEV_DIR=/ceph/build.ceph-dev/run/"${HOSTNAME}"/dev +export_var CEPH_OUT_DIR=/ceph/build.ceph-dev/run/"${HOSTNAME}"/out export_var CEPH_PORT="${CEPH_PORT:-10000}" rm -rf /ceph/build + mkdir -p "${CEPH_CONF_PATH}" if [[ "${IS_CEPH_RPM}" == 1 ]]; then - export_var CEPH_BIN=/usr/bin + export_var CEPH_BIN=/usr/bin/ export_var CEPH_LIB=/usr/lib64/ceph export_var EC_PATH="$CEPH_LIB"/erasure-code export_var OBJCLASS_PATH=/usr/lib64/rados-classes - + ln -sf "$EC_PATH"/* "$CEPH_LIB" ln -sf "$OBJCLASS_PATH"/* "$CEPH_LIB" diff --git a/docker/ceph/set-monitoring.sh b/docker/ceph/set-monitoring.sh index 3015f6bed59..dc724535759 100755 --- a/docker/ceph/set-monitoring.sh +++ b/docker/ceph/set-monitoring.sh @@ -1,14 +1,15 @@ #!/bin/bash set -e +source /docker/set-mstart-env.sh +echo $CEPH1 # Configure grafana set_grafana_api_url() { while true; do GRAFANA_IP=$(getent ahosts grafana | tail -1 | awk '{print $1}') if [[ -n "$GRAFANA_IP" ]]; then - "$CEPH_BIN"/ceph dashboard set-grafana-api-url "https://$GRAFANA_IP:$GRAFANA_HOST_PORT" - + CEPH_CLI dashboard set-grafana-api-url "https://$GRAFANA_IP:$GRAFANA_HOST_PORT" return fi @@ -22,8 +23,7 @@ set_alertmanager_api_host() { while true; do ALERTMANAGER_IP=$(getent ahosts alertmanager | tail -1 | awk '{print $1}') if [[ -n "$ALERTMANAGER_IP" ]]; then - "$CEPH_BIN"/ceph dashboard set-alertmanager-api-host "http://$ALERTMANAGER_IP:$ALERTMANAGER_HOST_PORT" - + CEPH_CLI dashboard set-alertmanager-api-host "http://$ALERTMANAGER_IP:$ALERTMANAGER_HOST_PORT" return fi @@ -37,8 +37,7 @@ set_prometheus_api_host() { while true; do PROMETHEUS_IP=$(getent ahosts prometheus | tail -1 | awk '{print $1}') if [[ -n "$PROMETHEUS_IP" ]]; then - "$CEPH_BIN"/ceph dashboard set-prometheus-api-host "http://$PROMETHEUS_IP:$PROMETHEUS_HOST_PORT" - + CEPH_CLI dashboard set-prometheus-api-host "http://$PROMETHEUS_IP:$PROMETHEUS_HOST_PORT" return fi diff --git a/docker/ceph/set-mstart-env.sh b/docker/ceph/set-mstart-env.sh new file mode 100755 index 00000000000..b0627706611 --- /dev/null +++ b/docker/ceph/set-mstart-env.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -e + +export_var() { + export "${1?}" + echo "export $1" >> /root/.bashrc +} + +echo $NUMBER_OF_CLUSTERS +declare -a CEPH_BIN_LIST +declare -a CEPH_CLUSTERS +for (( cluster=1; cluster<=$NUMBER_OF_CLUSTERS; cluster++ )); do + CEPH_CLUSTERS[cluster]="CEPH${cluster}" + CEPH_BIN_LIST[cluster]=${CEPH_CLUSTERS[cluster]} + echo ${CEPH_CLUSTERS[cluster]} + export_var ${CEPH_BIN_LIST[cluster]}="/ceph/src/mrun ${CEPH_CLUSTERS[cluster]} ceph" +done + +CEPH_CLI() { + export_var CEPH_CONF_PATH=/ceph/build.ceph-dev/run/CEPH1/ + /ceph/src/mrun CEPH1 ceph "$@" +} + +CEPH2_CLI() { + export_var CEPH_CONF_PATH=/ceph/build.ceph-dev/run/CEPH2/ + /ceph/src/mrun CEPH2 ceph "$@" +} + +CEPH_CLI_ALL() { + for cluster in ${CEPH_BIN_LIST[@]}; do + export_var CEPH_BIN=/usr/bin/ + export_var BUILD_DIR=/ceph/build.ceph-dev/ + export_var CEPH_CONF_PATH=/ceph/build.ceph-dev/run/"${cluster}/" + + cd /ceph/build.ceph-dev + ${!cluster} $@ + done +} diff --git a/docker/ceph/set-start-env.sh b/docker/ceph/set-start-env.sh index c940e37e2a3..03623b818e7 100755 --- a/docker/ceph/set-start-env.sh +++ b/docker/ceph/set-start-env.sh @@ -2,6 +2,8 @@ set -e +source /docker/set-mstart-env.sh + [[ -z "$MGR" ]] && export MGR=1 [[ -z "$MGR_PYTHON_PATH" ]] && export MGR_PYTHON_PATH=/ceph/src/pybind/mgr [[ -d "$MGR_PYTHON_PATH"/dashboard/frontend ]] && export IS_UPSTREAM_LUMINOUS=0 diff --git a/docker/ceph/start-ceph.sh b/docker/ceph/start-ceph.sh index ee5d304a4f8..af53b2a2efb 100755 --- a/docker/ceph/start-ceph.sh +++ b/docker/ceph/start-ceph.sh @@ -4,12 +4,13 @@ set -e source /docker/set-start-env.sh -# Build frontend: +#Build frontend: if [[ "$FRONTEND_BUILD_REQUIRED" == 1 ]]; then cd "$MGR_PYTHON_PATH"/dashboard/frontend # Set dev server proxy: TARGET_URL="${HTTP_PROTO}://${HOSTNAME}:${CEPH_MGR_DASHBOARD_PORT}" + echo "target" $TARGET_URL [[ -n "${DASHBOARD_URL}" ]] && TARGET_URL=${DASHBOARD_URL} jq "(.[] | .target)=\""${TARGET_URL}"\"" proxy.conf.json.sample > proxy.conf.json @@ -18,7 +19,7 @@ if [[ "$FRONTEND_BUILD_REQUIRED" == 1 ]]; then npm update @angular/cli fi - npm ci + npm i if [[ -z "${DASHBOARD_URL}" ]]; then # Required to run dashboard python module. @@ -40,14 +41,28 @@ fi rm -rf "$CEPH_CONF_PATH"/* -cd /ceph/build -../src/vstart.sh ${VSTART_OPTIONS} +for cluster in ${CEPH_CLUSTERS[@]}; do + cd /ceph/build + echo ${cluster} + # Env. vars used in vstart. + export_var CEPH_ASOK_DIR=/ceph/build.ceph-dev/run/"${cluster}"/asok + export_var CEPH_CONF=/ceph/build.ceph-dev/run/"${cluster}"/ceph.conf + export_var CEPH_CONF_PATH=/ceph/build.ceph-dev/run/"${cluster}" + export_var CEPH_DEV_DIR=/ceph/build.ceph-dev/run/"${cluster}"/dev + export_var CEPH_OUT_DIR=/ceph/build.ceph-dev/run/"${cluster}"/out + export VSTART_DEST=/ceph/build.ceph-dev/run/"${cluster}" + export CEPH_MGR_DASHBOARD_PORT=$(($CEPH_PORT + 1000)) + + ../src/mstart.sh ${cluster} ${VSTART_OPTIONS} + + export CEPH_PORT=$(($CEPH_PORT + 100)) +done echo 'vstart.sh completed!' # Create rbd pool: -"$CEPH_BIN"/ceph osd pool create rbd 8 8 replicated -"$CEPH_BIN"/ceph osd pool application enable rbd rbd +CEPH_CLI_ALL osd pool create rbd 8 8 replicated +CEPH_CLI_ALL osd pool application enable rbd rbd # Configure Object Gateway: if [[ "$RGW" -gt 0 || "$RGW_MULTISITE" == 1 ]]; then @@ -56,7 +71,7 @@ fi # Enable prometheus module if [[ "$IS_FIRST_CLUSTER" == 1 ]]; then - "$CEPH_BIN"/ceph mgr module enable prometheus + CEPH_CLI mgr module enable prometheus echo 'Prometheus mgr module enabled.' fi @@ -75,8 +90,8 @@ if [[ "$DASHBOARD_SSL" == 0 && "$VSTART_HAS_SSL_FLAG" == 0 && "$IS_FIRST_CLUSTER SSL_OPTIONS='' fi - "$CEPH_BIN"/ceph config set mgr mgr/dashboard/ssl false $SSL_OPTIONS - "$CEPH_BIN"/ceph config set mgr mgr/dashboard/x/server_port "$CEPH_MGR_DASHBOARD_PORT" $SSL_OPTIONS + CEPH_CLI_ALL config set mgr mgr/dashboard/ssl false $SSL_OPTIONS + CEPH_CLI_ALL config set mgr mgr/dashboard/x/server_port "$CEPH_MGR_DASHBOARD_PORT" $SSL_OPTIONS /docker/restart-dashboard.sh echo "SSL disabled." @@ -90,12 +105,12 @@ fi create_test_user() { DASHBOARD_TEST_USER_SECRET_FILE="/tmp/dashboard-test-user-secret.txt" printf 'test' > "${DASHBOARD_TEST_USER_SECRET_FILE}" - "$CEPH_BIN"/ceph dashboard ac-user-create test -i "${DASHBOARD_TEST_USER_SECRET_FILE}" "${DASHBOARD_USER_CREATE_OPTIONS}" + CEPH_CLI_ALL dashboard ac-user-create test -i "${DASHBOARD_TEST_USER_SECRET_FILE}" "${DASHBOARD_USER_CREATE_OPTIONS}" } -create_test_user || "$CEPH_BIN"/ceph dashboard ac-user-create test test "${DASHBOARD_USER_CREATE_OPTIONS}" +create_test_user || CEPH_CLI_ALL dashboard ac-user-create test test "${DASHBOARD_USER_CREATE_OPTIONS}" # Enable debug mode. -"$CEPH_BIN"/ceph dashboard debug enable +CEPH_CLI_ALL dashboard debug enable # Set monitoring stack: /docker/set-monitoring.sh @@ -104,4 +119,4 @@ create_test_user || "$CEPH_BIN"/ceph dashboard ac-user-create test test "${DASHB [[ "$CEPH_VERSION" -le '14' ]] && exit 0 # Set dashboard log level. -"$CEPH_BIN"/ceph config set mgr mgr/dashboard/log_level debug +CEPH_CLI_ALL config set mgr mgr/dashboard/log_level debug