Skip to content

move check to script in docker #29

move check to script in docker

move check to script in docker #29

name: Splunkconf backup app Integration Test
on:
push:
branches: [main, develop]
pull_request:
branches: [main]
workflow_dispatch:
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
APP_NAME: "splunkconf-backup"
APP_DEPLOY_PATH: "/opt/splunk/etc/apps"
OUTPUT_DIR: "/opt/splunk/var/backups" # Replace with the specific directory to monitor
WAIT_TIME: 300 # Seconds to wait after restart for file generation
SPLUNK_PASSWORD: "Chang3d!" # Splunk admin password
jobs:
test-splunk-app:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
splunk_version:
- "10.2"
- "10.0"
- "9.4"
- "9.3"
- "9.2"
- "9.1"
- "9.0"
name: "Splunk ${{ matrix.splunk_version }} : Testing backups"
steps:
# -------------------------------------------------------
# Step 1: Checkout the repository
# -------------------------------------------------------
- name: show disk usage before checkout
run: |
echo "=== Disk usage ==="
df -h .
du -sh .
echo ""
- name: Checkout repository (sparse - splunkconf-backup only)
uses: actions/checkout@v6
with:
sparse-checkout: |
splunkconf-backup
sparse-checkout-cone-mode: true
- name: Verify checkout and show disk usage
run: |
echo "=== Checked out files ==="
find . -type f | head -50
echo ""
echo "=== Disk usage ==="
df -h .
du -sh .
echo ""
echo "=== App directory ==="
ls -la splunkconf-backup/ || { echo "ERROR: splunkconf-backup directory not found"; exit 1; }
# -------------------------------------------------------
# Step 2: Prepare the app package with test files
# -------------------------------------------------------
- name: Prepare app package
run: |
echo "=== Preparing app package ==="
mkdir -p app_package
# Copy the sparse-checkout app directory as the package
cp -r splunkconf-backup app_package/${{ env.APP_NAME }}
# Remove any git artifacts that might have been included
rm -rf "app_package/${{ env.APP_NAME }}/.git"
echo "=== App package contents ==="
find app_package/ -type f | head -50
echo ""
du -sh app_package/
echo "Package prepared successfully."
# -------------------------------------------------------
# Step 3: Install zstd (needed for .tar.zst verification)
# -------------------------------------------------------
- name: Install zstd
run: |
sudo apt-get update -qq
sudo apt-get install -y -qq zstd
# -------------------------------------------------------
# Step 4: Start Splunk container
# -------------------------------------------------------
- name: Start Splunk container
run: |
echo "=== Starting Splunk ${{ matrix.splunk_version }} ==="
docker run -d \
--name splunk \
--hostname splunk \
-p 8000:8000 \
-p 8089:8089 \
-e SPLUNK_GENERAL_TERMS="--accept-sgt-current-at-splunk-com" \
-e SPLUNK_START_ARGS="--accept-license" \
-e SPLUNK_PASSWORD="${{ env.SPLUNK_PASSWORD }}" \
-e TZ="Europe/Paris" \
splunk/splunk:${{ matrix.splunk_version }}
echo "Container started. Waiting for Splunk to become healthy..."
# -------------------------------------------------------
# Step 5: Wait for Splunk to be ready
# -------------------------------------------------------
- name: Wait for Splunk ${{ matrix.splunk_version }} to be ready
run: |
echo "=== Waiting for Splunk ${{ matrix.splunk_version }} to become ready ==="
MAX_RETRIES=60
RETRY_INTERVAL=10
RETRIES=0
while [ $RETRIES -lt $MAX_RETRIES ]; do
HEALTH=$(docker inspect --format='{{.State.Health.Status}}' splunk 2>/dev/null || echo "not_found")
if [ "$HEALTH" = "healthy" ]; then
echo "Splunk is healthy after $((RETRIES * RETRY_INTERVAL)) seconds."
break
fi
RETRIES=$((RETRIES + 1))
echo " Attempt ${RETRIES}/${MAX_RETRIES} - Status: ${HEALTH}. Retrying in ${RETRY_INTERVAL}s..."
sleep $RETRY_INTERVAL
done
if [ "$HEALTH" != "healthy" ]; then
echo "ERROR: Splunk did not become healthy within $((MAX_RETRIES * RETRY_INTERVAL)) seconds."
docker logs splunk
exit 1
fi
# Additional check via REST API
HTTP_CODE=$(docker exec splunk curl -s -o /dev/null -w "%{http_code}" \
-k -u admin:${{ env.SPLUNK_PASSWORD }} \
https://localhost:8089/services/server/info)
if [ "$HTTP_CODE" -eq 200 ]; then
echo "Splunk REST API is responding (HTTP ${HTTP_CODE})."
else
echo "ERROR: Splunk REST API returned HTTP ${HTTP_CODE}."
exit 1
fi
# -------------------------------------------------------
# Step 6: Deploy the app into Splunk
# -------------------------------------------------------
- name: Deploy ${{ env.APP_NAME }} app to Splunk
run: |
echo "=== Deploying app to Splunk ==="
# Copy the app package into the Splunk container
# app is inside a git subdirectory with same folder name
docker cp app_package/${{ env.APP_NAME }} \
splunk:${{ env.APP_DEPLOY_PATH }}/${{ env.APP_NAME }}
# Create local directory and add configuration file
echo "=== Creating local configuration ==="
docker exec --user root splunk mkdir -p \
${{ env.APP_DEPLOY_PATH }}/${{ env.APP_NAME }}/local
docker exec --user root splunk bash -c 'cat > ${{ env.APP_DEPLOY_PATH }}/${{ env.APP_NAME }}/local/splunkconf-backup.conf << EOF
[settings]
DEBUG=1
EOF'
echo "=== Fixing app permission, giving files to Splunk ==="
# Fix ownership inside the container
# run as root to be allowed to chown
docker exec --user root splunk chown -R splunk:splunk \
${{ env.APP_DEPLOY_PATH }}/${{ env.APP_NAME }}
# Verify deployment
echo "=== Verifying app deployment ==="
docker exec --user splunk splunk ls -la ${{ env.APP_DEPLOY_PATH }}/${{ env.APP_NAME }}/
echo "App deployed successfully."
# -------------------------------------------------------
# Step 7: Restart Splunk to load the app
# -------------------------------------------------------
- name: Restart Splunk
run: |
echo "=== Restarting Splunk to load ${{ env.APP_NAME }} app ==="
echo "Checking id"
docker exec splunk id
echo "listing processes in splunk docker"
docker exec splunk ps aux
docker exec --user splunk splunk cat /opt/splunk/etc/splunk-launch.conf
echo "restarting splunk as splunk user"
docker exec --user splunk splunk /opt/splunk/bin/splunk restart
# docker exec --user splunk splunk /opt/splunk/bin/splunk restart \
# -auth admin:${{ env.SPLUNK_PASSWORD }}
echo "Restart command issued. Waiting for Splunk to come back..."
# Wait for Splunk to be healthy again after restart
MAX_RETRIES=60
RETRY_INTERVAL=10
RETRIES=0
while [ $RETRIES -lt $MAX_RETRIES ]; do
docker exec --user splunk splunk ls /opt/splunk/var
docker exec --user splunk splunk ls -l /opt/splunk/var/backups
HEALTH=$(docker inspect --format='{{.State.Health.Status}}' splunk 2>/dev/null || echo "not_found")
if [ "$HEALTH" = "healthy" ]; then
echo "Splunk is healthy again after $((RETRIES * RETRY_INTERVAL)) seconds."
break
fi
RETRIES=$((RETRIES + 1))
echo " Attempt ${RETRIES}/${MAX_RETRIES} - Status: ${HEALTH}. Retrying in ${RETRY_INTERVAL}s..."
sleep $RETRY_INTERVAL
done
echo "after restart : listing processes in splunk docker"
docker exec splunk ps aux
if [ "$HEALTH" != "healthy" ]; then
echo "ERROR: Splunk did not recover after restart."
docker logs splunk
exit 1
fi
# -------------------------------------------------------
# Step 8: Verify app is loaded in Splunk
# -------------------------------------------------------
- name: Verify app is loaded
run: |
echo "=== Verifying app is loaded in Splunk ==="
APP_STATUS=$(docker exec --user splunk splunk /opt/splunk/bin/splunk display app ${{ env.APP_NAME }} \
-auth admin:${{ env.SPLUNK_PASSWORD }} 2>&1 || true)
echo "App status: ${APP_STATUS}"
# Also check via REST API
HTTP_CODE=$(docker exec splunk curl -s -o /dev/null -w "%{http_code}" \
-k -u admin:${{ env.SPLUNK_PASSWORD }} \
https://localhost:8089/servicesNS/nobody/${{ env.APP_NAME }}/configs/conf-app/ui)
echo "REST API check for app: HTTP ${HTTP_CODE}"
# -------------------------------------------------------
# Step 9: Wait for file generation
# -------------------------------------------------------

Check failure on line 260 in .github/workflows/splunkconf-backup-test.yml

View workflow run for this annotation

GitHub Actions / .github/workflows/splunkconf-backup-test.yml

Invalid workflow file

You have an error in your yaml syntax on line 260
- name: Wait for output files to be generated
run: |
echo "=== Waiting up to ${{ env.WAIT_TIME }} seconds for files to be generated ==="
echo "Start time: $(date)"
echo "Early exit condition: at least 2 backup files found"
echo ""
TOTAL_WAIT=${{ env.WAIT_TIME }}
ELAPSED=0
CHECK_INTERVAL=15
REQUIRED_FILE_COUNT=2
STABLE_AGE_SECONDS=5
# Create the helper script inside the container for reliable execution
docker exec --user splunk splunk bash -c 'cat > /tmp/count_stable_files.sh << '\''SCRIPT'\''
#!/bin/bash
OUTPUT_DIR="$1"
STABLE_AGE="$2"
NOW=$(date +%s)
TOTAL=0
STABLE=0
while IFS= read -r f; do
[ -z "$f" ] && continue
TOTAL=$((TOTAL + 1))
MTIME=$(stat -c %Y "$f" 2>/dev/null || echo "0")
AGE=$((NOW - MTIME))
if [ "$AGE" -ge "$STABLE_AGE" ]; then
STABLE=$((STABLE + 1))
fi
done < <(find "$OUTPUT_DIR" -type f \( -name "*.tar" -o -name "*.tar.gz" -o -name "*.tgz" -o -name "*.tar.zst" \) 2>/dev/null)
echo "${TOTAL} ${STABLE}"
SCRIPT'
docker exec --user splunk splunk chmod +x /tmp/count_stable_files.sh
# # Helper script to count files not modified in the last N seconds
# # This runs inside the container
# COUNT_STABLE_FILES='
# STABLE_AGE='"${STABLE_AGE_SECONDS}"'
# NOW=$(date +%s)
# COUNT=0
# find '"${{ env.OUTPUT_DIR }}"' \( -name "*.tar" -o -name "*.tar.gz" -o -name "*.tgz" -o -name "*.tar.zst" \) -type f 2>/dev/null | while read -r f; do
# MTIME=$(stat -c %Y "$f" 2>/dev/null || echo "0")
# AGE=$((NOW - MTIME))
# if [ "$AGE" -ge "$STABLE_AGE" ]; then
# COUNT=$((COUNT + 1))
# fi
# echo "$COUNT"
# done | tail -1
# '
while [ $ELAPSED -lt $TOTAL_WAIT ]; do
sleep $CHECK_INTERVAL
ELAPSED=$((ELAPSED + CHECK_INTERVAL))
docker exec --user splunk splunk ls /opt/splunk/var
docker exec --user splunk splunk ls -l /opt/splunk/var/backups
# Get total and stable counts
COUNTS=$(docker exec --user splunk splunk bash /tmp/count_stable_files.sh \
"${{ env.OUTPUT_DIR }}" "$STABLE_AGE_SECONDS" 2>/dev/null || echo "0 0")
TOTAL_COUNT=$(echo "$COUNTS" | awk '{print $1}')
STABLE_COUNT=$(echo "$COUNTS" | awk '{print $2}')
# Handle empty results
TOTAL_COUNT=${TOTAL_COUNT:-0}
STABLE_COUNT=${STABLE_COUNT:-0}
echo " [${ELAPSED}s/${TOTAL_WAIT}s] Total files: ${TOTAL_COUNT} | Stable (>${STABLE_AGE_SECONDS}s): ${STABLE_COUNT}"
if [ "$STABLE_COUNT" -ge $REQUIRED_FILE_COUNT ]; then
echo ""
echo " 🚀 Early exit: ${STABLE_COUNT} stable file(s) found (required: ${REQUIRED_FILE_COUNT})"
break
fi
done
echo ""
echo "End time: $(date)"
echo "Total wait: ${ELAPSED} seconds"
# Final count
FINAL_COUNT=$(docker exec --user splunk splunk bash -c \
"find ${{ env.OUTPUT_DIR }} \( -name '*.tar' -o -name '*.tar.gz' -o -name '*.tgz' -o -name '*.tar.zst' \) 2>/dev/null | wc -l" \
|| echo "0")
echo "Final file count: ${FINAL_COUNT}"
if [ "$FINAL_COUNT" -lt $REQUIRED_FILE_COUNT ]; then
echo "⚠️ WARNING: Only ${FINAL_COUNT} file(s) found after full wait. Expected at least ${REQUIRED_FILE_COUNT}."
else
echo "✅ ${FINAL_COUNT} backup file(s) detected."
fi
# -------------------------------------------------------
# Step 10: Check for file creation in the output directory
# -------------------------------------------------------
- name: Verify files were created
run: |
echo "=== Checking for generated files in ${{ env.OUTPUT_DIR }} dir==="
# List everything in the output directory
echo "--- Full directory listing ---"
docker exec --user splunk splunk bash -c \
"find ${{ env.OUTPUT_DIR }} -type f -ls 2>/dev/null" || true
# Count tar files specifically
FILE_COUNT=$(docker exec --user splunk splunk bash -c \
"find ${{ env.OUTPUT_DIR }} -type f \( -name '*.tar' -o -name '*.tar.gz' -o -name '*.tgz' -o -name '*.tar.zst' \) 2>/dev/null | wc -l")
echo "Total backup files found: ${FILE_COUNT}"
if [ "$FILE_COUNT" -eq 0 ]; then
echo "ERROR: No backup files were generated in ${{ env.OUTPUT_DIR }}."
echo ""
echo "--- splunkconf-backup log ---"
docker exec --user splunk splunk cat /opt/splunk/var/log/splunk/splunkconf-backup.log 2>/dev/null || true
echo ""
echo "--- Splunk logs for debugging ---"
docker exec --user splunk splunk tail -100 /opt/splunk/var/log/splunk/splunkd.log || true
exit 1
fi
echo "SUCCESS: ${FILE_COUNT} backup file(s) found."
# -------------------------------------------------------
# Step 11: Download (copy) the generated tar files
# -------------------------------------------------------
- name: Download generated backup files from container
run: |
echo "=== Downloading backup files from Splunk container ==="
mkdir -p ./test_output
# Copy all backup files from the output directory
docker exec --user splunk splunk bash -c \
"find ${{ env.OUTPUT_DIR }} -type f \( -name '*.tar' -o -name '*.tar.gz' -o -name '*.tgz' -o -name '*.tar.zst' \)" \
| while read -r filepath; do
filename=$(basename "$filepath")
echo " Downloading: ${filepath} -> ./test_output/${filename}"
docker cp "splunk:${filepath}" "./test_output/${filename}"
done
echo ""
echo "=== Downloaded files ==="
ls -la ./test_output/
# -------------------------------------------------------
# Step 12: Verify tar file contents
# -------------------------------------------------------
- name: Verify tar file contents
run: |
echo "=== Verifying tar file contents ==="
# Define expected files per backup type
# ---- ETC backups ----
# Filename pattern: backupconfsplunk-*-etc-targeted-*.tar.gz (or .tar.zst)
EXPECTED_FILES_ETC=(
"./etc/apps/splunkconf-backup/lookups/splunkconf-backup-expected.csv"
"./etc/apps/journald_input/default/inputs.conf"
"./etc/apps/splunk_metrics_workspace/README/workspace.conf.spec"
)
# ---- STATE backups ----
# Filename pattern: backupconfsplunk-*-state-*.tar.gz (or .tar.zst)
EXPECTED_FILES_STATE=(
"./var/lib/splunk/fishbucket/db/"
"./var/run/splunk/confsnapshot/"
)
OVERALL_RESULT=0
ETC_FOUND=0
STATE_FOUND=0
# Helper function to list tar contents based on file extension
list_tar_contents() {
local tarfile="$1"
case "$tarfile" in
*.tar.zst)
tar -I zstd -tf "$tarfile" 2>/dev/null
;;
*.tar.gz|*.tgz)
tar -tzf "$tarfile" 2>/dev/null
;;
*.tar)
tar -tf "$tarfile" 2>/dev/null
;;
*)
# Try plain, then gzip, then zstd
tar -tf "$tarfile" 2>/dev/null || \
tar -tzf "$tarfile" 2>/dev/null || \
tar -I zstd -tf "$tarfile" 2>/dev/null
;;
esac
}
# Helper function to detect backup type from filename
get_backup_type() {
local filename="$1"
if echo "$filename" | grep -qE 'backupconfsplunk-.*-etc-targeted-'; then
echo "etc"
elif echo "$filename" | grep -qE 'backupconfsplunk-.*-state-'; then
echo "state"
else
echo "unknown"
fi
}
for tarfile in ./test_output/*.tar ./test_output/*.tar.gz ./test_output/*.tgz ./test_output/*.tar.zst; do
# Skip if glob didn't match any files
[ -e "$tarfile" ] || continue
FILENAME=$(basename "$tarfile")
BACKUP_TYPE=$(get_backup_type "$FILENAME")
echo ""
echo "============================================"
echo "Inspecting: ${FILENAME}"
echo "Detected backup type: ${BACKUP_TYPE}"
echo "============================================"
# Select the appropriate expected files list
case "$BACKUP_TYPE" in
etc)
EXPECTED_FILES=("${EXPECTED_FILES_ETC[@]}")
ETC_FOUND=1
;;
state)
EXPECTED_FILES=("${EXPECTED_FILES_STATE[@]}")
STATE_FOUND=1
;;
unknown)
echo " ⚠️ WARNING: Unknown backup type for file: ${FILENAME}"
echo " Skipping content verification for this file."
echo " Expected patterns:"
echo " - ETC: backupconfsplunk-*-etc-targeted-*.tar.gz (or .tar.zst)"
echo " - STATE: backupconfsplunk-*-state-*.tar.gz (or .tar.zst)"
continue
;;
esac
echo "--- Archive contents ---"
TAR_CONTENTS=$(list_tar_contents "$tarfile") || {
echo "ERROR: Unable to read file: $tarfile"
OVERALL_RESULT=1
continue
}
echo "$TAR_CONTENTS"
echo ""
echo "--- Checking expected files for ${BACKUP_TYPE} backup ---"
for expected in "${EXPECTED_FILES[@]}"; do
if echo "$TAR_CONTENTS" | grep -q "$expected"; then
echo " ✅ FOUND: $expected"
else
echo " ❌ MISSING: $expected"
OVERALL_RESULT=1
fi
done
done
# Verify that we found at least one of each backup type
echo ""
echo "============================================"
echo "Backup type coverage check:"
echo "============================================"
if [ "$ETC_FOUND" -eq 1 ]; then
echo " ✅ ETC backup found and verified"
else
echo " ❌ No ETC backup found (expected: backupconfsplunk-*-etc-targeted-*)"
OVERALL_RESULT=1
fi
if [ "$STATE_FOUND" -eq 1 ]; then
echo " ✅ STATE backup found and verified"
else
echo " ❌ No STATE backup found (expected: backupconfsplunk-*-state-*)"
OVERALL_RESULT=1
fi
echo ""
echo "============================================"
if [ $OVERALL_RESULT -eq 0 ]; then
echo "🎉 ALL VERIFICATIONS PASSED"
else
echo "💥 SOME VERIFICATIONS FAILED"
exit 1
fi
# -------------------------------------------------------
# Step 13: Upload test artifacts (for debugging)
# -------------------------------------------------------
- name: Upload test artifacts
if: always()
uses: actions/upload-artifact@v6
with:
name: splunk-test-output-${{ matrix.splunk_version }}
path: ./test_output/
retention-days: 7
# -------------------------------------------------------
# Step 14: Collect Splunk logs on failure
# -------------------------------------------------------
- name: Collect Splunk logs on failure
if: failure()
run: |
echo "=== Collecting Splunk logs for debugging ==="
mkdir -p ./debug_logs
docker exec --user splunk splunk cat /opt/splunk/var/log/splunk/splunkconf-backup.log \
> ./debug_logs/splunkconf-backup.log 2>/dev/null || true
docker exec --user splunk splunk cat /opt/splunk/var/log/splunk/splunkd.log \
> ./debug_logs/splunkd.log 2>/dev/null || true
docker exec --user splunk splunk cat /opt/splunk/var/log/splunk/splunk_stderr.log \
> ./debug_logs/splunk_stderr.log 2>/dev/null || true
docker exec --user splunk splunk cat /opt/splunk/var/log/splunk/python.log \
> ./debug_logs/python.log 2>/dev/null || true
docker logs splunk \
> ./debug_logs/container.log 2>&1 || true
echo "Logs collected."
- name: Upload debug logs on failure
if: failure()
uses: actions/upload-artifact@v6
with:
name: splunk-debug-logs-${{ matrix.splunk_version }}
path: ./debug_logs/
retention-days: 7
# -------------------------------------------------------
# Step 15: Cleanup
# -------------------------------------------------------
- name: Cleanup Splunk container
if: always()
run: |
docker stop splunk 2>/dev/null || true
docker rm splunk 2>/dev/null || true
echo "Cleanup complete."