Updated onboarding script

This commit is contained in:
Ben D.
2026-04-23 14:35:08 -07:00
parent 8b18dbcddb
commit e1f647445f

View File

@@ -1,45 +1,91 @@
#!/bin/bash
set -e
# ─── Usage ───────────────────────────────────────────────────────────────────
# =============================================================================
# onboard_client.sh
# MSP client onboarding script — creates inventory, Semaphore project,
# SSH keys, and task templates.
#
# Required: --id, --name, --slug
# Everything else is optional and can be added later via --update
#
# Client types (--type):
# basic — preflight + linux patch only (no hypervisor)
# linux — same as basic (alias)
# proxmox — basic + proxmox upgrade/backup templates
# xcpng — basic + xcp-ng pool upgrade/snapshot templates
# firewall — preflight only (for firewall-only clients)
# custom — no templates created, add manually
#
# Idempotency: re-running against an existing client slug will detect the
# existing project and skip creation steps. Use --update to add optional
# config (e.g. add Proxmox to a previously basic client).
# =============================================================================
usage() {
echo "Usage: $0 [options]"
echo ""
echo "Options:"
echo " -i, --id Client ID (e.g. DFA-001)"
echo " -n, --name Client name (e.g. 'DFA Tech Colo')"
echo " -s, --slug Short slug for filenames (e.g. dfa_tech)"
echo " -w, --webhook n8n webhook URL"
echo " -b, --billing Billing model (hybrid|per-host|time-saved|tiered) default: hybrid"
echo " -e, --estimate Human time estimate seconds (default: 2700)"
echo " -v, --vpn VPN type (ipsec|openvpn) default: ipsec"
echo " -H, --hypervisor Hypervisor type (proxmox|xcpng|baremetal|mixed) default: proxmox"
echo " --proxmox-host Proxmox host IP or hostname (required for proxmox/mixed)"
echo " --proxmox-token-id Proxmox API token ID (e.g. ansible@pve!token-name)"
echo " --proxmox-token-secret Proxmox API token secret"
echo " --bootstrap-proxmox Create Proxmox role/user/token via API (prompts for root password)"
echo " --proxmox-role Proxmox role name to create (default: AnsibleMSP)"
echo " --proxmox-user Proxmox user to create (default: ansible@pve)"
echo " --proxmox-token-name Token name to create (default: ansible-token)"
echo " --xo-url XCP-NG/XO API URL (default: \$XO_URL from env)"
echo " --xo-token XCP-NG/XO API token (default: \$XO_TOKEN from env)"
echo " --semaphore-url Semaphore base URL (default: http://localhost:3000)"
echo " --semaphore-token Semaphore API token"
echo " --gitea-url Gitea repo SSH URL"
echo " --project-name Semaphore project name (default: 'Client - <n>')"
echo " --dry-run Print actions without executing"
echo ""
echo "Example:"
echo " $0 -i ACME-001 -n 'Acme Corp' -s acme_corp \\"
echo " -H proxmox --proxmox-host 192.168.1.10 --bootstrap-proxmox"
cat << EOF
Usage: $0 [options]
Required:
-i, --id Client ID (e.g. RP-001)
-n, --name Client name (e.g. 'Royal Pizza')
-s, --slug Short slug for filenames (e.g. royal_pizza)
Client type (controls which templates are created):
-t, --type Client type: basic|linux|proxmox|xcpng|firewall|custom
default: basic
Optional:
-w, --webhook n8n webhook URL
-b, --billing Billing model (hybrid|per-host|time-saved|tiered) default: hybrid
-e, --estimate Human time estimate in seconds default: 2700
-v, --vpn VPN type (ipsec|openvpn|none) default: none
Hypervisor options (proxmox type):
--proxmox-host Proxmox API host IP
--proxmox-token-id Proxmox API token ID (e.g. ansible@pve!token-name)
--proxmox-token-secret Proxmox API token secret
Hypervisor options (xcpng type):
--xo-url XenOrchestra URL (e.g. https://xoa.example.com)
--xo-token XenOrchestra API token
Semaphore/Gitea (usually not needed — read from defaults):
--semaphore-url Semaphore base URL (default: http://localhost:3000)
--semaphore-token Semaphore API token
--gitea-url Gitea repo SSH URL
--project-name Override Semaphore project name
Modes:
--update Update an existing client project (add config, add templates)
--dry-run Show what would be done without making changes
Examples:
# Minimal — firewall-only client
$0 -i RP-001 -n 'Royal Pizza' -s royal_pizza --type firewall
# Basic Linux client with webhook
$0 -i RP-001 -n 'Royal Pizza' -s royal_pizza -w https://n8n.voice1.me/webhook/xxx
# Full Proxmox client
$0 -i RP-001 -n 'Royal Pizza' -s royal_pizza --type proxmox \\
--proxmox-host 10.1.2.3 --proxmox-token-id ansible@pve!token \\
--proxmox-token-secret abc123 -w https://n8n.voice1.me/webhook/xxx
# Add Proxmox to an existing basic client
$0 -i RP-001 -n 'Royal Pizza' -s royal_pizza --type proxmox --update \\
--proxmox-host 10.1.2.3 --proxmox-token-id ansible@pve!token \\
--proxmox-token-secret abc123
EOF
exit 1
}
# ─── Defaults ────────────────────────────────────────────────────────────────
CLIENT_TYPE="basic"
BILLING="hybrid"
ESTIMATE="2700"
VPN="ipsec"
HYPERVISOR="proxmox"
VPN="none"
SEMAPHORE_URL="http://localhost:3000"
REPO_DIR="/opt/ansible-msp-automations"
GITEA_DEPLOY_KEY="/root/.ssh/gitea_ansible"
@@ -47,13 +93,11 @@ GITEA_REPO_URL="ssh://git@172.31.10.8:2222/VOICE1/ansible-msp-automations.git"
PROXMOX_HOST=""
PROXMOX_TOKEN_ID=""
PROXMOX_TOKEN_SECRET=""
BOOTSTRAP_PROXMOX=false
PROXMOX_ROLE="AnsibleMSP"
PROXMOX_USER="ansible@pve"
PROXMOX_TOKEN_NAME="ansible-token"
XO_URL_ARG=""
XO_TOKEN_ARG=""
XO_URL=""
XO_TOKEN=""
WEBHOOK_URL=""
PROJECT_NAME_OVERRIDE=""
UPDATE_MODE=false
DRY_RUN=false
# ─── Parse args ──────────────────────────────────────────────────────────────
@@ -62,66 +106,57 @@ while [[ $# -gt 0 ]]; do
-i|--id) CLIENT_ID="$2"; shift 2 ;;
-n|--name) CLIENT_NAME="$2"; shift 2 ;;
-s|--slug) CLIENT_SLUG="$2"; shift 2 ;;
-t|--type) CLIENT_TYPE="$2"; shift 2 ;;
-w|--webhook) WEBHOOK_URL="$2"; shift 2 ;;
-b|--billing) BILLING="$2"; shift 2 ;;
-e|--estimate) ESTIMATE="$2"; shift 2 ;;
-v|--vpn) VPN="$2"; shift 2 ;;
-H|--hypervisor) HYPERVISOR="$2"; shift 2 ;;
--proxmox-host) PROXMOX_HOST="$2"; shift 2 ;;
--proxmox-token-id) PROXMOX_TOKEN_ID="$2"; shift 2 ;;
--proxmox-token-secret) PROXMOX_TOKEN_SECRET="$2"; shift 2 ;;
--bootstrap-proxmox) BOOTSTRAP_PROXMOX=true; shift ;;
--proxmox-role) PROXMOX_ROLE="$2"; shift 2 ;;
--proxmox-user) PROXMOX_USER="$2"; shift 2 ;;
--proxmox-token-name) PROXMOX_TOKEN_NAME="$2"; shift 2 ;;
--xo-url) XO_URL_ARG="$2"; shift 2 ;;
--xo-token) XO_TOKEN_ARG="$2"; shift 2 ;;
--xo-url) XO_URL="$2"; shift 2 ;;
--xo-token) XO_TOKEN="$2"; shift 2 ;;
--semaphore-url) SEMAPHORE_URL="$2"; shift 2 ;;
--semaphore-token) SEMAPHORE_TOKEN="$2"; shift 2 ;;
--gitea-url) GITEA_REPO_URL="$2"; shift 2 ;;
--project-name) PROJECT_NAME_OVERRIDE="$2"; shift 2 ;;
--update) UPDATE_MODE=true; shift ;;
--dry-run) DRY_RUN=true; shift ;;
*) usage ;;
-h|--help) usage ;;
*) echo "Unknown option: $1"; usage ;;
esac
done
# ─── Validate required args ──────────────────────────────────────────────────
if [[ -z "${CLIENT_ID:-}" || -z "${CLIENT_NAME:-}" || -z "${CLIENT_SLUG:-}" ]]; then
echo "ERROR: --id, --name, and --slug are required"
echo "ERROR: --id, --name, and --slug are required."
usage
fi
# Load env file if present
if [[ -f "/root/.semaphore_env" ]]; then
# Validate client type
case "$CLIENT_TYPE" in
basic|linux|proxmox|xcpng|firewall|custom) ;;
*) echo "ERROR: Invalid --type '$CLIENT_TYPE'. Must be: basic|linux|proxmox|xcpng|firewall|custom"; exit 1 ;;
esac
# Normalize 'linux' as alias for 'basic'
[[ "$CLIENT_TYPE" == "linux" ]] && CLIENT_TYPE="basic"
# Warn if proxmox type but no credentials provided
if [[ "$CLIENT_TYPE" == "proxmox" && -z "$PROXMOX_HOST" ]]; then
echo " ⚠ --type proxmox specified but --proxmox-host not set."
echo " Proxmox templates will be created but credentials will be empty."
echo " Re-run with --update --proxmox-host ... to add credentials later."
fi
# Load SEMAPHORE_TOKEN from env file if not passed
if [[ -z "${SEMAPHORE_TOKEN:-}" && -f "/root/.semaphore_env" ]]; then
source /root/.semaphore_env
fi
# Resolve webhook — arg > env
WEBHOOK_URL="${WEBHOOK_URL:-${N8N_WEBHOOK_URL:-}}"
if [[ -z "${WEBHOOK_URL:-}" ]]; then
echo "ERROR: --webhook is required (or set N8N_WEBHOOK_URL in /root/.semaphore_env)"
usage
fi
# Resolve XO vars — arg > env
RESOLVED_XO_URL="${XO_URL_ARG:-${XO_URL:-}}"
RESOLVED_XO_TOKEN="${XO_TOKEN_ARG:-${XO_TOKEN:-}}"
# Validate XO vars for xcpng/mixed
if [[ "$HYPERVISOR" == "xcpng" || "$HYPERVISOR" == "mixed" ]]; then
if [[ -z "$RESOLVED_XO_URL" || -z "$RESOLVED_XO_TOKEN" ]]; then
echo "ERROR: --xo-url and --xo-token are required for hypervisor type '$HYPERVISOR'"
echo " (or set XO_URL and XO_TOKEN in /root/.semaphore_env)"
exit 1
fi
fi
# Validate proxmox host for bootstrap or proxmox hypervisor
if [[ "$BOOTSTRAP_PROXMOX" == true || "$HYPERVISOR" == "proxmox" || "$HYPERVISOR" == "mixed" ]]; then
if [[ -z "$PROXMOX_HOST" ]]; then
echo "ERROR: --proxmox-host is required for hypervisor type '$HYPERVISOR'"
exit 1
fi
if [[ -z "${SEMAPHORE_TOKEN:-}" ]]; then
echo "ERROR: No Semaphore token. Set SEMAPHORE_TOKEN in /root/.semaphore_env or pass --semaphore-token"
exit 1
fi
PROJECT_NAME="${PROJECT_NAME_OVERRIDE:-Client - ${CLIENT_NAME}}"
@@ -129,191 +164,77 @@ KEY_FILE="/root/.ssh/client_${CLIENT_SLUG}"
INVENTORY_DIR="$REPO_DIR/inventories/client_${CLIENT_SLUG}"
INVENTORY_REPO_PATH="inventories/client_${CLIENT_SLUG}/hosts.yml"
# Determine hypervisor type for inventory
case "$CLIENT_TYPE" in
proxmox) HYPERVISOR="proxmox" ;;
xcpng) HYPERVISOR="xcpng" ;;
*) HYPERVISOR="none" ;;
esac
# ─── Dry run header ──────────────────────────────────────────────────────────
if $DRY_RUN; then
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " DRY RUN — no changes will be made"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
fi
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " Onboarding client: $CLIENT_NAME ($CLIENT_ID)"
[[ "$DRY_RUN" == true ]] && echo " ⚠ DRY RUN — no changes will be made"
if $UPDATE_MODE; then
echo " Updating client: $CLIENT_NAME ($CLIENT_ID)"
else
echo " Onboarding client: $CLIENT_NAME ($CLIENT_ID)"
fi
echo " Type: $CLIENT_TYPE | VPN: $VPN | Billing: $BILLING"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# ─── Step 0: Bootstrap Proxmox (optional) ────────────────────────────────────
if [[ "$BOOTSTRAP_PROXMOX" == true ]]; then
echo ""
echo "[ 0 ] Bootstrapping Proxmox user/role/token on $PROXMOX_HOST..."
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo " [dry-run] Would authenticate as root@pam and create:"
echo " Role: $PROXMOX_ROLE"
echo " User: $PROXMOX_USER"
echo " Token: $PROXMOX_USER!$PROXMOX_TOKEN_NAME"
echo " Permission: / (propagate=1)"
# ─── Helper: semaphore API call ───────────────────────────────────────────────
semaphore_api() {
local METHOD="$1"
local PATH="$2"
local DATA="${3:-}"
if [[ -n "$DATA" ]]; then
curl -s -X "$METHOD" "$SEMAPHORE_URL/api$PATH" \
-H "Authorization: Bearer $SEMAPHORE_TOKEN" \
-H "Content-Type: application/json" \
-d "$DATA"
else
# Prompt for root password
echo -n " Proxmox root password for $PROXMOX_HOST: "
read -rs PVE_ROOT_PASS
echo ""
PVE_API="https://${PROXMOX_HOST}:8006/api2/json"
# Get auth ticket
echo " Authenticating as root@pam..."
TICKET_RESPONSE=$(curl -sk -X POST "$PVE_API/access/ticket" \
-d "username=root@pam" \
--data-urlencode "password=$PVE_ROOT_PASS")
PVE_TICKET=$(echo "$TICKET_RESPONSE" | jq -r '.data.ticket // empty')
PVE_CSRF=$(echo "$TICKET_RESPONSE" | jq -r '.data.CSRFPreventionToken // empty')
if [[ -z "$PVE_TICKET" ]]; then
echo " ✗ Authentication failed — check root password"
echo " Response: $TICKET_RESPONSE"
exit 1
fi
echo " ✓ Authenticated"
# Helper functions — use ticket auth for all subsequent calls
pve_post() {
local endpoint="$1"; shift
curl -sk -X POST "$PVE_API/$endpoint" \
-H "CSRFPreventionToken: $PVE_CSRF" \
-b "PVEAuthCookie=$PVE_TICKET" \
"$@"
}
pve_put() {
local endpoint="$1"; shift
curl -sk -X PUT "$PVE_API/$endpoint" \
-H "CSRFPreventionToken: $PVE_CSRF" \
-b "PVEAuthCookie=$PVE_TICKET" \
"$@"
}
# Create role
echo " Creating role: $PROXMOX_ROLE..."
pve_post "access/roles" \
-d "roleid=$PROXMOX_ROLE" \
-d "privs=VM.Snapshot,VM.Snapshot.Rollback,VM.Audit,VM.PowerMgmt,Datastore.AllocateSpace" \
> /dev/null
echo " ✓ Role created (or already exists): $PROXMOX_ROLE"
# Create user
PVE_USERID_ENCODED=$(python3 -c "import urllib.parse; print(urllib.parse.quote('$PROXMOX_USER'))")
echo " Creating user: $PROXMOX_USER..."
USER_RESPONSE=$(pve_post "access/users" \
-d "userid=$PROXMOX_USER" \
-d "password=ChangeMe123!" \
-d "comment=Ansible MSP automation user — managed by onboard_client.sh")
USER_DATA=$(echo "$USER_RESPONSE" | jq -r '.data // empty')
USER_ERR=$(echo "$USER_RESPONSE" | jq -r '.errors // empty')
if [[ -n "$USER_ERR" ]]; then
if echo "$USER_RESPONSE" | jq -r '.errors | to_entries[].value' 2>/dev/null | grep -qi "already exist"; then
echo " ⚠ User already exists — continuing"
else
echo " ✗ Failed to create user"
echo " Response: $USER_RESPONSE"
exit 1
fi
else
echo " ✓ User created: $PROXMOX_USER"
echo " ⚠ Temporary password set: ChangeMe123! — rotate after token creation"
fi
# Create API token
echo " Creating API token: $PROXMOX_TOKEN_NAME..."
TOKEN_RESPONSE=$(pve_post "access/users/${PVE_USERID_ENCODED}/token/$PROXMOX_TOKEN_NAME" \
-d "privsep=0" \
-d "comment=Ansible MSP snapshot token — managed by onboard_client.sh")
TOKEN_VALUE=$(echo "$TOKEN_RESPONSE" | jq -r '.data.value // empty')
if [[ -z "$TOKEN_VALUE" ]]; then
echo " ✗ Failed to create token (may already exist — token secrets are shown only once)"
echo " Response: $TOKEN_RESPONSE"
echo ""
echo " If the token already exists, delete it in Proxmox UI and re-run --bootstrap-proxmox"
echo " Or pass --proxmox-token-id and --proxmox-token-secret directly to skip bootstrap"
exit 1
fi
PROXMOX_TOKEN_ID="${PROXMOX_USER}!${PROXMOX_TOKEN_NAME}"
PROXMOX_TOKEN_SECRET="$TOKEN_VALUE"
echo " ✓ Token created"
echo ""
echo " ┌─ SAVE THIS TOKEN SECRET — shown only once ─────────────────────"
echo " │"
echo " │ Token ID: $PROXMOX_TOKEN_ID"
echo " │ Token Secret: $PROXMOX_TOKEN_SECRET"
echo " │"
echo " └────────────────────────────────────────────────────────────────"
echo ""
# Assign permission: user + role to path /
echo " Assigning permission: $PROXMOX_USER + $PROXMOX_ROLE → / (propagate)..."
pve_put "access/acl" \
-d "path=/" \
-d "users=$PROXMOX_USER" \
-d "roles=$PROXMOX_ROLE" \
-d "propagate=1" \
> /dev/null
echo " ✓ Permission assigned"
echo ""
echo " ✓ Proxmox bootstrap complete"
curl -s -X "$METHOD" "$SEMAPHORE_URL/api$PATH" \
-H "Authorization: Bearer $SEMAPHORE_TOKEN"
fi
fi
}
# ─── Validate token args (post-bootstrap) ────────────────────────────────────
if [[ "$BOOTSTRAP_PROXMOX" != true ]]; then
if [[ "$HYPERVISOR" == "proxmox" || "$HYPERVISOR" == "mixed" ]]; then
if [[ -z "$PROXMOX_TOKEN_ID" || -z "$PROXMOX_TOKEN_SECRET" ]]; then
echo "ERROR: --proxmox-token-id and --proxmox-token-secret are required"
echo " (or use --bootstrap-proxmox to create them automatically)"
exit 1
fi
fi
fi
# ─── Step 1: Generate SSH key ─────────────────────────────────────────────────
# ─── Step 1: SSH key ─────────────────────────────────────────────────────────
echo ""
echo "[ 1/6 ] Generating SSH key..."
if [[ "$DRY_RUN" == true ]]; then
echo " [dry-run] Would generate SSH key at $KEY_FILE"
elif [[ -f "$KEY_FILE" ]]; then
echo "[ 1 ] SSH key..."
if [[ -f "$KEY_FILE" ]]; then
echo " ⚠ Key already exists at $KEY_FILE — skipping generation"
elif $DRY_RUN; then
echo " [dry-run] Would generate SSH key at $KEY_FILE"
else
ssh-keygen -t ed25519 -C "ansible-${CLIENT_SLUG}" -f "$KEY_FILE" -N ""
echo " ✓ Key generated: $KEY_FILE"
fi
if [[ "$DRY_RUN" != true ]]; then
echo ""
echo " ┌─ Deploy this public key to all client hosts ───────────────────────"
echo " │"
sed 's/^/ │ /' "$KEY_FILE.pub"
echo " │"
echo " └────────────────────────────────────────────────────────────────────"
fi
# ─── Step 2: Create inventory from template ───────────────────────────────────
echo ""
echo "[ 2/6 ] Creating inventory..."
echo " ┌─ Deploy this public key to all client hosts ───────────────────────"
echo " │"
[[ -f "$KEY_FILE.pub" ]] && sed 's/^/ │ /' "$KEY_FILE.pub" || echo " │ (key not yet generated)"
echo " │"
echo " └────────────────────────────────────────────────────────────────────"
if [[ "$DRY_RUN" == true ]]; then
# ─── Step 2: Inventory ───────────────────────────────────────────────────────
echo ""
echo "[ 2 ] Inventory..."
if [[ -d "$INVENTORY_DIR" && ! $UPDATE_MODE ]]; then
echo " ⚠ Inventory already exists at $INVENTORY_DIR — skipping"
elif $DRY_RUN; then
echo " [dry-run] Would create inventory at $INVENTORY_DIR"
elif [[ -d "$INVENTORY_DIR" ]]; then
echo " ⚠ Inventory directory already exists — skipping"
else
cp -r "$REPO_DIR/inventories/client_template" "$INVENTORY_DIR"
mkdir -p "$INVENTORY_DIR/group_vars"
XCPNG_HOSTS_BLOCK=""
if [[ "$HYPERVISOR" == "xcpng" || "$HYPERVISOR" == "mixed" ]]; then
XCPNG_HOSTS_BLOCK="
xcpng_hosts:
hosts: {}
vars:
ansible_connection: local"
fi
cat > "$INVENTORY_DIR/hosts.yml" << HOSTSEOF
---
all:
@@ -334,9 +255,8 @@ all:
linux_hosts:
hosts: {}
vars:
ansible_user: ansible-msp-agent
ansible_become: true
ansible_become_method: sudo
ansible_user: root
os_family: "debian"
windows_hosts:
hosts: {}
@@ -346,15 +266,14 @@ all:
ansible_winrm_transport: ntlm
ansible_winrm_server_cert_validation: validate
ansible_port: 5986
${XCPNG_HOSTS_BLOCK}
HOSTSEOF
cat > "$INVENTORY_DIR/group_vars/all.yml" << VARSEOF
---
# Client: ${CLIENT_NAME} (${CLIENT_ID})
# Onboarded: $(date +%Y-%m-%d)
# Type: ${CLIENT_TYPE}
# VPN: ${VPN}
# Hypervisor: ${HYPERVISOR}
# Billing: ${BILLING}
# Add client-specific overrides below
@@ -363,251 +282,274 @@ VARSEOF
echo " ✓ Inventory created at $INVENTORY_DIR"
fi
# ─── Step 3: Commit and push to Gitea ────────────────────────────────────────
# ─── Step 3: Commit to Gitea ─────────────────────────────────────────────────
echo ""
echo "[ 3/6 ] Committing to Gitea..."
if [[ "$DRY_RUN" == true ]]; then
echo " [dry-run] Would commit and push inventory to Gitea"
echo "[ 3 ] Gitea..."
if $DRY_RUN; then
echo " [dry-run] Would commit and push to Gitea"
else
cd "$REPO_DIR"
git add .
git commit -m "Onboard client: ${CLIENT_NAME} (${CLIENT_ID}) — inventory scaffold" \
|| echo " ⚠ Nothing to commit"
git push origin main
echo " ✓ Pushed to Gitea"
git diff --cached --quiet && echo " ⚠ Nothing to commit" || {
git commit -m "Onboard client: ${CLIENT_NAME} (${CLIENT_ID}) — ${CLIENT_TYPE} inventory scaffold"
git push origin main
echo " ✓ Pushed to Gitea"
}
fi
# ─── Step 4: Create Semaphore project via API ─────────────────────────────────
# ─── Step 4: Semaphore project ───────────────────────────────────────────────
echo ""
echo "[ 4/6 ] Creating Semaphore project via API..."
echo "[ 4 ] Semaphore project..."
if [[ "$DRY_RUN" == true ]]; then
# Check if project already exists
EXISTING_PROJECT=$(semaphore_api GET "/projects" | jq -r ".[] | select(.name == \"$PROJECT_NAME\") | .id")
if [[ -n "$EXISTING_PROJECT" && ! $UPDATE_MODE ]]; then
echo " ⚠ Project '$PROJECT_NAME' already exists (ID: $EXISTING_PROJECT) — skipping creation"
echo " Run with --update to add config to the existing project"
PROJECT_ID="$EXISTING_PROJECT"
elif $DRY_RUN; then
echo " [dry-run] Would create Semaphore project: $PROJECT_NAME"
echo " [dry-run] Would create keys, repo, environment, inventory, and templates"
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " [dry-run] Onboarding simulation complete: $CLIENT_NAME"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
if [[ -z "${SEMAPHORE_TOKEN:-}" ]]; then
echo " ✗ No semaphore token available."
echo " Set SEMAPHORE_TOKEN in /root/.semaphore_env or pass --semaphore-token"
exit 1
fi
# 4a. Create project
PROJECT_RESPONSE=$(curl -s -X POST "$SEMAPHORE_URL/api/projects" \
-H "Authorization: Bearer $SEMAPHORE_TOKEN" \
-H "Content-Type: application/json" \
-d "{\"name\": \"$PROJECT_NAME\", \"alert\": false, \"max_parallel_tasks\": 0}")
PROJECT_ID=$(echo "$PROJECT_RESPONSE" | jq -r '.id')
if [[ "$PROJECT_ID" == "null" || -z "$PROJECT_ID" ]]; then
echo " ✗ Failed to create project"
echo " Response: $PROJECT_RESPONSE"
exit 1
fi
echo " ✓ Project created: $PROJECT_NAME (ID: $PROJECT_ID)"
# 4b. Add gitea-deploy key
GITEA_KEY_RESPONSE=$(curl -s -X POST "$SEMAPHORE_URL/api/project/$PROJECT_ID/keys" \
-H "Authorization: Bearer $SEMAPHORE_TOKEN" \
-H "Content-Type: application/json" \
-d "{
\"name\": \"gitea-deploy\",
\"type\": \"ssh\",
\"project_id\": $PROJECT_ID,
\"ssh\": {
\"login\": \"\",
\"passphrase\": \"\",
\"private_key\": $(jq -Rs . < "$GITEA_DEPLOY_KEY")
}
}")
GITEA_KEY_ID=$(echo "$GITEA_KEY_RESPONSE" | jq -r '.id')
echo " ✓ gitea-deploy key added (ID: $GITEA_KEY_ID)"
# 4c. Add client SSH key
CLIENT_KEY_RESPONSE=$(curl -s -X POST "$SEMAPHORE_URL/api/project/$PROJECT_ID/keys" \
-H "Authorization: Bearer $SEMAPHORE_TOKEN" \
-H "Content-Type: application/json" \
-d "{
\"name\": \"client-${CLIENT_SLUG}-ssh\",
\"type\": \"ssh\",
\"project_id\": $PROJECT_ID,
\"ssh\": {
\"login\": \"\",
\"passphrase\": \"\",
\"private_key\": $(jq -Rs . < "$KEY_FILE")
}
}")
CLIENT_KEY_ID=$(echo "$CLIENT_KEY_RESPONSE" | jq -r '.id')
echo " ✓ Client SSH key added (ID: $CLIENT_KEY_ID)"
# 4d. Add None key
NONE_KEY_RESPONSE=$(curl -s -X POST "$SEMAPHORE_URL/api/project/$PROJECT_ID/keys" \
-H "Authorization: Bearer $SEMAPHORE_TOKEN" \
-H "Content-Type: application/json" \
-d "{
\"name\": \"None\",
\"type\": \"none\",
\"project_id\": $PROJECT_ID
}")
NONE_KEY_ID=$(echo "$NONE_KEY_RESPONSE" | jq -r '.id')
echo " ✓ None key added (ID: $NONE_KEY_ID)"
# 4e. Add repository
REPO_RESPONSE=$(curl -s -X POST "$SEMAPHORE_URL/api/project/$PROJECT_ID/repositories" \
-H "Authorization: Bearer $SEMAPHORE_TOKEN" \
-H "Content-Type: application/json" \
-d "{
\"name\": \"ansible-msp-automations\",
\"project_id\": $PROJECT_ID,
\"git_url\": \"$GITEA_REPO_URL\",
\"git_branch\": \"main\",
\"ssh_key_id\": $GITEA_KEY_ID
}")
REPO_ID=$(echo "$REPO_RESPONSE" | jq -r '.id')
echo " ✓ Repository linked (ID: $REPO_ID)"
# 4f. Build variable group JSON
if [[ "$HYPERVISOR" == "xcpng" || "$HYPERVISOR" == "mixed" ]]; then
VARS_JSON=$(jq -n \
--arg webhook "$WEBHOOK_URL" \
--arg cid "$CLIENT_ID" \
--arg cname "$CLIENT_NAME" \
--arg billing "$BILLING" \
--arg estimate "$ESTIMATE" \
--arg xo_url "$RESOLVED_XO_URL" \
--arg xo_token "$RESOLVED_XO_TOKEN" \
'{
N8N_WEBHOOK_URL: $webhook,
CLIENT_ID: $cid,
CLIENT_NAME: $cname,
BILLING_MODEL: $billing,
HUMAN_ESTIMATE_SECONDS: $estimate,
XO_URL: $xo_url,
XO_TOKEN: $xo_token
}')
PROJECT_ID="DRY_RUN"
else
if [[ -n "$EXISTING_PROJECT" ]]; then
echo " Update mode — using existing project (ID: $EXISTING_PROJECT)"
PROJECT_ID="$EXISTING_PROJECT"
else
PROJECT_RESPONSE=$(semaphore_api POST "/projects" \
"{\"name\": \"$PROJECT_NAME\", \"alert\": false, \"max_parallel_tasks\": 0}")
PROJECT_ID=$(echo "$PROJECT_RESPONSE" | jq -r '.id')
if [[ "$PROJECT_ID" == "null" || -z "$PROJECT_ID" ]]; then
echo " ✗ Failed to create project: $PROJECT_RESPONSE"
exit 1
fi
echo " ✓ Project created: $PROJECT_NAME (ID: $PROJECT_ID)"
fi
# Keys — check if they exist first to avoid duplicates on --update
EXISTING_KEYS=$(semaphore_api GET "/project/$PROJECT_ID/keys")
HAS_GITEA=$(echo "$EXISTING_KEYS" | jq -r '.[] | select(.name == "gitea-deploy") | .id')
if [[ -z "$HAS_GITEA" ]]; then
GITEA_KEY_RESPONSE=$(semaphore_api POST "/project/$PROJECT_ID/keys" \
"{\"name\": \"gitea-deploy\", \"type\": \"ssh\", \"project_id\": $PROJECT_ID,
\"ssh\": {\"login\": \"\", \"passphrase\": \"\", \"private_key\": $(jq -Rs . < "$GITEA_DEPLOY_KEY")}}")
GITEA_KEY_ID=$(echo "$GITEA_KEY_RESPONSE" | jq -r '.id')
echo " ✓ gitea-deploy key added (ID: $GITEA_KEY_ID)"
else
GITEA_KEY_ID="$HAS_GITEA"
echo " ⚠ gitea-deploy key already exists (ID: $GITEA_KEY_ID)"
fi
HAS_CLIENT_KEY=$(echo "$EXISTING_KEYS" | jq -r ".[] | select(.name == \"client-${CLIENT_SLUG}-ssh\") | .id")
if [[ -z "$HAS_CLIENT_KEY" ]]; then
CLIENT_KEY_RESPONSE=$(semaphore_api POST "/project/$PROJECT_ID/keys" \
"{\"name\": \"client-${CLIENT_SLUG}-ssh\", \"type\": \"ssh\", \"project_id\": $PROJECT_ID,
\"ssh\": {\"login\": \"\", \"passphrase\": \"\", \"private_key\": $(jq -Rs . < "$KEY_FILE")}}")
CLIENT_KEY_ID=$(echo "$CLIENT_KEY_RESPONSE" | jq -r '.id')
echo " ✓ Client SSH key added (ID: $CLIENT_KEY_ID)"
else
CLIENT_KEY_ID="$HAS_CLIENT_KEY"
echo " ⚠ Client SSH key already exists (ID: $CLIENT_KEY_ID)"
fi
HAS_NONE=$(echo "$EXISTING_KEYS" | jq -r '.[] | select(.name == "None") | .id')
if [[ -z "$HAS_NONE" ]]; then
NONE_KEY_RESPONSE=$(semaphore_api POST "/project/$PROJECT_ID/keys" \
"{\"name\": \"None\", \"type\": \"none\", \"project_id\": $PROJECT_ID}")
NONE_KEY_ID=$(echo "$NONE_KEY_RESPONSE" | jq -r '.id')
echo " ✓ None key added (ID: $NONE_KEY_ID)"
else
NONE_KEY_ID="$HAS_NONE"
echo " ⚠ None key already exists (ID: $NONE_KEY_ID)"
fi
# Repository
HAS_REPO=$(semaphore_api GET "/project/$PROJECT_ID/repositories" | jq -r '.[] | select(.name == "ansible-msp-automations") | .id')
if [[ -z "$HAS_REPO" ]]; then
REPO_RESPONSE=$(semaphore_api POST "/project/$PROJECT_ID/repositories" \
"{\"name\": \"ansible-msp-automations\", \"project_id\": $PROJECT_ID,
\"git_url\": \"$GITEA_REPO_URL\", \"git_branch\": \"main\",
\"ssh_key_id\": $GITEA_KEY_ID}")
REPO_ID=$(echo "$REPO_RESPONSE" | jq -r '.id')
echo " ✓ Repository linked (ID: $REPO_ID)"
else
REPO_ID="$HAS_REPO"
echo " ⚠ Repository already linked (ID: $REPO_ID)"
fi
# Environment / variable group
# Build vars JSON — only include non-empty values
VARS_JSON=$(jq -n \
--arg webhook "$WEBHOOK_URL" \
--arg cid "$CLIENT_ID" \
--arg cname "$CLIENT_NAME" \
--arg billing "$BILLING" \
--arg estimate "$ESTIMATE" \
--arg phost "$PROXMOX_HOST" \
--arg ptid "$PROXMOX_TOKEN_ID" \
--arg ptsecret "$PROXMOX_TOKEN_SECRET" \
--arg webhook "$WEBHOOK_URL" \
--arg cid "$CLIENT_ID" \
--arg cname "$CLIENT_NAME" \
--arg billing "$BILLING" \
--arg estimate "$ESTIMATE" \
'{
N8N_WEBHOOK_URL: $webhook,
CLIENT_ID: $cid,
CLIENT_NAME: $cname,
BILLING_MODEL: $billing,
HUMAN_ESTIMATE_SECONDS: $estimate,
PROXMOX_HOST: $phost,
PROXMOX_TOKEN_ID: $ptid,
PROXMOX_TOKEN_SECRET: $ptsecret
HUMAN_ESTIMATE_SECONDS: $estimate
}')
# Add webhook only if provided
[[ -n "$WEBHOOK_URL" ]] && VARS_JSON=$(echo "$VARS_JSON" | jq --arg v "$WEBHOOK_URL" '. + {N8N_WEBHOOK_URL: $v}')
# Add Proxmox vars only if provided
[[ -n "$PROXMOX_HOST" ]] && VARS_JSON=$(echo "$VARS_JSON" | jq --arg v "$PROXMOX_HOST" '. + {PROXMOX_HOST: $v}')
[[ -n "$PROXMOX_TOKEN_ID" ]] && VARS_JSON=$(echo "$VARS_JSON" | jq --arg v "$PROXMOX_TOKEN_ID" '. + {PROXMOX_TOKEN_ID: $v}')
[[ -n "$PROXMOX_TOKEN_SECRET" ]] && VARS_JSON=$(echo "$VARS_JSON" | jq --arg v "$PROXMOX_TOKEN_SECRET" '. + {PROXMOX_TOKEN_SECRET: $v}')
# Add XO vars only if provided
[[ -n "$XO_URL" ]] && VARS_JSON=$(echo "$VARS_JSON" | jq --arg v "$XO_URL" '. + {XO_URL: $v}')
[[ -n "$XO_TOKEN" ]] && VARS_JSON=$(echo "$VARS_JSON" | jq --arg v "$XO_TOKEN" '. + {XO_TOKEN: $v}')
HAS_ENV=$(semaphore_api GET "/project/$PROJECT_ID/environment" | jq -r ".[] | select(.name == \"${CLIENT_SLUG}-vars\") | .id")
if [[ -z "$HAS_ENV" ]]; then
ENV_RESPONSE=$(semaphore_api POST "/project/$PROJECT_ID/environment" \
"{\"name\": \"${CLIENT_SLUG}-vars\", \"project_id\": $PROJECT_ID,
\"json\": $(echo "$VARS_JSON" | jq -Rs .), \"env\": \"{}\"}")
ENV_ID=$(echo "$ENV_RESPONSE" | jq -r '.id')
echo " ✓ Variable group created (ID: $ENV_ID)"
else
ENV_ID="$HAS_ENV"
echo " ⚠ Variable group already exists (ID: $ENV_ID) — updating..."
semaphore_api PUT "/project/$PROJECT_ID/environment/$ENV_ID" \
"{\"id\": $ENV_ID, \"name\": \"${CLIENT_SLUG}-vars\", \"project_id\": $PROJECT_ID,
\"json\": $(echo "$VARS_JSON" | jq -Rs .), \"env\": \"{}\"}" > /dev/null
echo " ✓ Variable group updated"
fi
# Inventory
HAS_INVENTORY=$(semaphore_api GET "/project/$PROJECT_ID/inventory" | jq -r ".[] | select(.name == \"client-${CLIENT_SLUG}\") | .id")
if [[ -z "$HAS_INVENTORY" ]]; then
INVENTORY_RESPONSE=$(semaphore_api POST "/project/$PROJECT_ID/inventory" \
"{\"name\": \"client-${CLIENT_SLUG}\", \"project_id\": $PROJECT_ID,
\"inventory\": \"$INVENTORY_REPO_PATH\",
\"ssh_key_id\": $CLIENT_KEY_ID,
\"become_key_id\": $NONE_KEY_ID,
\"type\": \"file\",
\"repository_id\": $REPO_ID}")
INVENTORY_ID=$(echo "$INVENTORY_RESPONSE" | jq -r '.id')
echo " ✓ Inventory created (ID: $INVENTORY_ID)"
else
INVENTORY_ID="$HAS_INVENTORY"
echo " ⚠ Inventory already exists (ID: $INVENTORY_ID)"
fi
fi
ENV_RESPONSE=$(curl -s -X POST "$SEMAPHORE_URL/api/project/$PROJECT_ID/environment" \
-H "Authorization: Bearer $SEMAPHORE_TOKEN" \
-H "Content-Type: application/json" \
-d "{
\"name\": \"${CLIENT_SLUG}-vars\",
\"project_id\": $PROJECT_ID,
\"json\": $(echo "$VARS_JSON" | jq -Rs .),
\"env\": \"{}\"
}")
ENV_ID=$(echo "$ENV_RESPONSE" | jq -r '.id')
echo " ✓ Variable group created (ID: $ENV_ID)"
# 4g. Add inventory
INVENTORY_RESPONSE=$(curl -s -X POST "$SEMAPHORE_URL/api/project/$PROJECT_ID/inventory" \
-H "Authorization: Bearer $SEMAPHORE_TOKEN" \
-H "Content-Type: application/json" \
-d "{
\"name\": \"client-${CLIENT_SLUG}\",
\"project_id\": $PROJECT_ID,
\"inventory\": \"$INVENTORY_REPO_PATH\",
\"ssh_key_id\": $CLIENT_KEY_ID,
\"become_key_id\": $NONE_KEY_ID,
\"type\": \"file\",
\"repository_id\": $REPO_ID
}")
INVENTORY_ID=$(echo "$INVENTORY_RESPONSE" | jq -r '.id')
echo " ✓ Inventory created (ID: $INVENTORY_ID)"
# ─── Step 5: Create task templates ───────────────────────────────────────────
# ─── Step 5: Task templates ──────────────────────────────────────────────────
echo ""
echo "[ 5/6 ] Creating task templates..."
echo "[ 5 ] Task templates..."
create_template() {
local TNAME="$1"
local PLAYBOOK="$2"
local DESC="$3"
local EXTRA_ARGS="${4:-[]}"
RESPONSE=$(curl -s -X POST "$SEMAPHORE_URL/api/project/$PROJECT_ID/templates" \
-H "Authorization: Bearer $SEMAPHORE_TOKEN" \
-H "Content-Type: application/json" \
-d "{
\"project_id\": $PROJECT_ID,
\"inventory_id\": $INVENTORY_ID,
\"repository_id\": $REPO_ID,
\"environment_id\": $ENV_ID,
\"name\": \"$TNAME\",
\"playbook\": \"$PLAYBOOK\",
\"arguments\": \"$EXTRA_ARGS\",
\"allow_override_args_in_task\": true,
\"description\": \"$DESC\",
\"app\": \"ansible\"
}")
local EXTRA_ARGS="${4:-}"
$DRY_RUN && { echo " [dry-run] Would create template: $TNAME"; return; }
[[ "$PROJECT_ID" == "DRY_RUN" ]] && return
# Skip if template already exists
EXISTING=$(semaphore_api GET "/project/$PROJECT_ID/templates" | jq -r ".[] | select(.name == \"$TNAME\") | .id")
if [[ -n "$EXISTING" ]]; then
echo " Template '$TNAME' already exists (ID: $EXISTING) — skipping"
return
fi
ARGS_JSON="[]"
[[ -n "$EXTRA_ARGS" ]] && ARGS_JSON="[\"$EXTRA_ARGS\"]"
RESPONSE=$(semaphore_api POST "/project/$PROJECT_ID/templates" \
"{\"project_id\": $PROJECT_ID, \"inventory_id\": $INVENTORY_ID,
\"repository_id\": $REPO_ID, \"environment_id\": $ENV_ID,
\"name\": \"$TNAME\", \"playbook\": \"$PLAYBOOK\",
\"arguments\": \"$ARGS_JSON\",
\"allow_override_args_in_task\": false,
\"description\": \"$DESC\", \"app\": \"ansible\"}")
echo "$(echo "$RESPONSE" | jq -r '"Template: \(.name) (ID: \(.id))"')"
}
create_template "Preflight Check" "playbooks/site_preflight.yml" "Run safety checks on all hosts before maintenance"
create_template "Linux Patch" "playbooks/linux_patch.yml" "Full Linux patch run with version tracking"
create_template "Full Maintenance" "playbooks/site_maintenance.yml" "Full maintenance: snapshot, preflight, patch"
create_template "Scheduled Reboot" "playbooks/linux_reboot.yml" "Reboot hosts that require it — pass -e force_reboot=true to reboot all"
if [[ "$HYPERVISOR" == "proxmox" || "$HYPERVISOR" == "mixed" ]]; then
create_template "Snapshot (Proxmox)" "playbooks/snapshot_pre.yml" "Pre-patch snapshot via Proxmox API"
# Templates common to all client types except 'custom'
if [[ "$CLIENT_TYPE" != "custom" ]]; then
create_template "Preflight Check" \
"playbooks/site_preflight.yml" \
"Run safety checks on all hosts before maintenance"
fi
if [[ "$HYPERVISOR" == "xcpng" || "$HYPERVISOR" == "mixed" ]]; then
create_template "Snapshot (XCP-NG)" "playbooks/snapshot_pre.yml" "Pre-patch snapshot via XO REST API"
create_template "XCP-NG Pool Update" "playbooks/xcpng_pool_update.yml" "Update XCP-NG host pool patches"
# Templates for clients with Linux hosts
if [[ "$CLIENT_TYPE" == "basic" || "$CLIENT_TYPE" == "proxmox" || "$CLIENT_TYPE" == "xcpng" ]]; then
create_template "Linux Patch" \
"playbooks/linux_patch.yml" \
"Full Linux patch run with version tracking"
create_template "Scheduled Reboot" \
"playbooks/linux_reboot.yml" \
"Reboot hosts that require it after kernel updates"
create_template "Full Maintenance" \
"playbooks/site_maintenance.yml" \
"Full maintenance: preflight, patch, reboot"
fi
# Proxmox-specific templates
if [[ "$CLIENT_TYPE" == "proxmox" ]]; then
create_template "Proxmox Status" \
"playbooks/proxmox_status.yml" \
"Cluster health report — nodes, VMs, CEPH, HA"
create_template "Proxmox Config Backup" \
"playbooks/proxmox_config_backup.yml" \
"Backup Proxmox node configs"
create_template "Proxmox Rolling Upgrade" \
"playbooks/proxmox_upgrade.yml" \
"Rolling upgrade of all Proxmox cluster nodes"
create_template "Proxmox Migrate VMs" \
"playbooks/proxmox_migrate_vms.yml" \
"Migrate VMs between nodes — drain/rebalance/restore/targeted"
create_template "Proxmox Snapshot" \
"playbooks/proxmox_snapshot.yml" \
"Pre/post maintenance VM snapshots"
fi
# XCP-NG-specific templates
if [[ "$CLIENT_TYPE" == "xcpng" ]]; then
create_template "XCP-NG Pool Upgrade" \
"playbooks/xcp_pool_upgrade.yml" \
"Rolling XCP-NG pool upgrade"
create_template "Snapshot (XCP-NG)" \
"playbooks/xcp_xo_vm_snapshot.yml" \
"Pre-maintenance VM snapshots via XenOrchestra"
create_template "Snapshot Cleanup" \
"playbooks/xcp_xo_snapshot_cleanup.yml" \
"Clean up old XCP-NG snapshots"
create_template "Migrate VMs (XCP-NG)" \
"playbooks/xcp_xo_vm_migrate.yml" \
"VM migration via XenOrchestra"
fi
# ─── Step 6: Summary ─────────────────────────────────────────────────────────
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " ✓ Client onboarding complete: $CLIENT_NAME"
if $DRY_RUN; then
echo " [dry-run complete] No changes made."
elif $UPDATE_MODE; then
echo " ✓ Client update complete: $CLIENT_NAME"
else
echo " ✓ Client onboarding complete: $CLIENT_NAME"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
echo " Semaphore project ID: $PROJECT_ID"
echo " Inventory file: $INVENTORY_DIR/hosts.yml"
if [[ "$BOOTSTRAP_PROXMOX" == true && "$DRY_RUN" != true ]]; then
echo ""
echo " Proxmox credentials (also stored in Semaphore variable group):"
echo " Token ID: $PROXMOX_TOKEN_ID"
echo " Token Secret: $PROXMOX_TOKEN_SECRET"
echo " ⚠ Rotate the $PROXMOX_USER password in Proxmox UI when convenient"
fi
[[ "$PROJECT_ID" != "DRY_RUN" ]] && echo " Semaphore project ID: $PROJECT_ID"
echo " Inventory: $INVENTORY_DIR/hosts.yml"
echo " Client type: $CLIENT_TYPE"
echo ""
echo " Next steps:"
echo " 1. Add hosts to: $INVENTORY_DIR/hosts.yml"
echo " 2. Run deploy_agent.sh to bootstrap ansible-msp-agent on Linux hosts"
echo " 3. git add . && git commit -m 'Add hosts for ${CLIENT_NAME}' && git push origin main"
echo " 4. Semaphore → $PROJECT_NAME → Preflight Check → ▶ Run"
echo " 5. Semaphore → $PROJECT_NAME → Full Maintenance → ▶ Run"
echo " 2. git add . && git commit -m 'Add hosts for ${CLIENT_NAME}' && git push origin main"
echo " 3. Run 'Preflight Check' in Semaphore to verify connectivity"
echo ""
echo " To reboot hosts after a deferred patch run:"
echo " Semaphore → $PROJECT_NAME → Scheduled Reboot → ▶ Run"
echo " (add extra var -e force_reboot=true to reboot all hosts regardless)"
echo ""
echo " Public key to deploy to client hosts:"
sed 's/^/ /' "$KEY_FILE.pub"
echo ""
if [[ -f "$KEY_FILE.pub" ]]; then
echo " Public key to deploy to client hosts:"
sed 's/^/ /' "$KEY_FILE.pub"
echo ""
fi