Copilot pipeline fix v2

.woodpecker.yml

@@ -109,25 +109,13 @@ steps:
              DEK_PATH=$(gcloud secrets versions access latest --secret=DEK_PATH_$ENV --project=$PROJECT); \
              export DEK_PATH; \
              export FROM_SECRETS_MANAGER=true; \
-             \
              # ── NEW: sync dev DEK into staging volume (uses Secret Manager) ── \
              if gcloud secrets describe WRAPPED_DEK_dev --project=$PROJECT >/dev/null 2>&1; then \
                echo \"🔁 Syncing dev DEK into staging volume\"; \
                gcloud secrets versions access latest --secret=WRAPPED_DEK_dev --project=$PROJECT > /tmp/dev_dek.enc; \
                if [ -s /tmp/dev_dek.enc ]; then \
                  docker volume ls -q | grep -qx aptiva_dek_staging || docker volume create aptiva_dek_staging >/dev/null; \
-                sudo docker run --rm -v aptiva_dek_staging:/v -v /tmp:/host busybox sh <<'EOF'
+                sudo docker run --rm -v aptiva_dek_staging:/v -v /tmp:/host busybox sh -c 'set -e; mkdir -p /v/staging; cp -f /host/dev_dek.enc /v/staging/dek.enc; chown 1000:1000 /v/staging/dek.enc; chmod 400 /v/staging/dek.enc; rm -f /v/staging/dek.fpr; echo -n "staging dek.enc bytes: "; wc -c </v/staging/dek.enc; ls -l /v/staging'
-                  set -e
-                  mkdir -p /v/staging
-                  cp -f /host/dev_dek.enc /v/staging/dek.enc
-                  chown 1000:1000 /v/staging/dek.enc
-                  chmod 400 /v/staging/dek.enc
-                  rm -f /v/staging/dek.fpr
-                  echo -n "staging dek.enc bytes: "; wc -c </v/staging/dek.enc
-                  ls -l /v/staging
-                  EOF
-
-                 \"; \
                else \
                  echo \"⚠️ WRAPPED_DEK_dev returned empty; skipping copy\"; \
                fi; \

Dockerfile.server1

@@ -1,15 +1,13 @@
 FROM node:20-bookworm-slim AS base
 
 RUN groupadd -r app && useradd -r -g app app
-
 WORKDIR /app
 
-# ---- native build deps ----
+# add curl for healthchecks (+ CA bundle)
 RUN apt-get update -y && \
     apt-get install -y --no-install-recommends \
-         build-essential python3 pkg-config && \
+      build-essential python3 pkg-config curl ca-certificates && \
     rm -rf /var/lib/apt/lists/*
-# ---------------------------
 
 COPY package*.json ./
 COPY public/ /app/public/
@@ -17,7 +15,6 @@ RUN npm ci --unsafe-perm
 COPY . .
 
 RUN mkdir -p /run/secrets && chown -R app:app /run/secrets
-
 USER app
 
 CMD ["node", "backend/server1.js"]

Dockerfile.server2

@@ -1,15 +1,13 @@
 FROM node:20-bookworm-slim AS base
 
 RUN groupadd -r app && useradd -r -g app app
-
 WORKDIR /app
 
-# ---- native build deps ----
+# add curl for healthchecks (+ CA bundle)
 RUN apt-get update -y && \
     apt-get install -y --no-install-recommends \
-         build-essential python3 pkg-config && \
+      build-essential python3 pkg-config curl ca-certificates && \
     rm -rf /var/lib/apt/lists/*
-# ---------------------------
 
 COPY package*.json ./
 COPY public/ /app/public/
@@ -17,6 +15,5 @@ RUN npm ci --unsafe-perm
 COPY . .
 
 RUN mkdir -p /run/secrets && chown -R app:app /run/secrets
-
 USER app
 CMD ["node", "backend/server2.js"]

Dockerfile.server3

@@ -1,15 +1,13 @@
 FROM node:20-bookworm-slim AS base
 
 RUN groupadd -r app && useradd -r -g app app
-
 WORKDIR /app
 
-# ---- native build deps ----
+# add curl for healthchecks (+ CA bundle)
 RUN apt-get update -y && \
     apt-get install -y --no-install-recommends \
-         build-essential python3 pkg-config && \
+      build-essential python3 pkg-config curl ca-certificates && \
     rm -rf /var/lib/apt/lists/*
-# ---------------------------
 
 COPY package*.json ./
 COPY public/ /app/public/
@@ -17,7 +15,6 @@ RUN npm ci --unsafe-perm
 COPY . .
 
 RUN mkdir -p /run/secrets && chown -R app:app /run/secrets
-
 USER app
 CMD ["node", "backend/server3.js"]
 

backend/server1.js

@@ -100,19 +100,75 @@ app.use(
   })
 );
 
- // LIVENESS: process is up; no external deps
+function fprPathFromEnv() {
- app.get('/healthz', (req, res) => res.type('text').send('OK'));
+  const p = (process.env.DEK_PATH || '').trim();
+  return p ? path.join(path.dirname(p), 'dek.fpr') : null;
+}
 
- // READINESS: KMS/DEK/DB are correct
+// 1) Liveness: process is up and event loop responsive
- app.get('/readyz', async (req, res) => {
+app.get('/livez', (_req, res) => res.type('text').send('OK'));
+
+// 2) Readiness: crypto + canary are good
+app.get('/readyz', async (_req, res) => {
   try {
-     await verifyCanary(pool);           // throws if bad
+    await initEncryption();      // load/unlock DEK
-     return res.type('text').send('READY');
+    await verifyCanary(pool);    // DB + decrypt sentinel
+    return res.type('text').send('OK');
   } catch (e) {
     console.error('[READYZ]', e.message);
-     return res.status(503).type('text').send('NOT_READY');
+    return res.status(500).type('text').send('FAIL');
   }
- });
+});
+
+// 3) Health: detailed JSON (you can curl this to “see everything”)
+app.get('/healthz', async (_req, res) => {
+  const out = {
+    service: process.env.npm_package_name || 'server',
+    version: process.env.IMG_TAG || null,
+    uptime_s: Math.floor(process.uptime()),
+    now: new Date().toISOString(),
+    checks: {
+      live: { ok: true },           // if we reached here, process is up
+      crypto: { ok: false, fp: null },
+      db: { ok: false, ping_ms: null },
+      canary: { ok: false }
+    }
+  };
+
+  // crypto / DEK
+  try {
+    await initEncryption();
+    out.checks.crypto.ok = true;
+    const p = fprPathFromEnv();
+    if (p) {
+      try { out.checks.crypto.fp = (await readFile(p, 'utf8')).trim(); }
+      catch { /* fp optional */ }
+    }
+  } catch (e) {
+    out.checks.crypto.error = e.message;
+  }
+
+  // DB ping
+  const t0 = Date.now();
+  try {
+    await pool.query('SELECT 1');
+    out.checks.db.ok = true;
+    out.checks.db.ping_ms = Date.now() - t0;
+  } catch (e) {
+    out.checks.db.error = e.message;
+  }
+
+  // canary
+  try {
+    await verifyCanary(pool);
+    out.checks.canary.ok = true;
+  } catch (e) {
+    out.checks.canary.error = e.message;
+  }
+
+  const ready = out.checks.crypto.ok && out.checks.db.ok && out.checks.canary.ok;
+  return res.status(ready ? 200 : 503).json(out);
+});
 
 // Enable CORS with dynamic origin checking
 app.use(

backend/server2.js

@@ -67,20 +67,75 @@ await verifyCanary(pool);
 const app = express();
 const PORT = process.env.SERVER2_PORT || 5001;
 
- // LIVENESS: process is up; no external deps
+function fprPathFromEnv() {
- app.get('/healthz', (req, res) => res.type('text').send('OK'));
+  const p = (process.env.DEK_PATH || '').trim();
+  return p ? path.join(path.dirname(p), 'dek.fpr') : null;
+}
 
- // READINESS: KMS/DEK/DB are correct
+// 1) Liveness: process is up and event loop responsive
- app.get('/readyz', async (req, res) => {
+app.get('/livez', (_req, res) => res.type('text').send('OK'));
+
+// 2) Readiness: crypto + canary are good
+app.get('/readyz', async (_req, res) => {
   try {
-     await verifyCanary(pool);           // throws if bad
+    await initEncryption();      // load/unlock DEK
-     return res.type('text').send('READY');
+    await verifyCanary(pool);    // DB + decrypt sentinel
+    return res.type('text').send('OK');
   } catch (e) {
     console.error('[READYZ]', e.message);
-     return res.status(503).type('text').send('NOT_READY');
+    return res.status(500).type('text').send('FAIL');
   }
- });
+});
 
+// 3) Health: detailed JSON (you can curl this to “see everything”)
+app.get('/healthz', async (_req, res) => {
+  const out = {
+    service: process.env.npm_package_name || 'server',
+    version: process.env.IMG_TAG || null,
+    uptime_s: Math.floor(process.uptime()),
+    now: new Date().toISOString(),
+    checks: {
+      live: { ok: true },           // if we reached here, process is up
+      crypto: { ok: false, fp: null },
+      db: { ok: false, ping_ms: null },
+      canary: { ok: false }
+    }
+  };
+
+  // crypto / DEK
+  try {
+    await initEncryption();
+    out.checks.crypto.ok = true;
+    const p = fprPathFromEnv();
+    if (p) {
+      try { out.checks.crypto.fp = (await readFile(p, 'utf8')).trim(); }
+      catch { /* fp optional */ }
+    }
+  } catch (e) {
+    out.checks.crypto.error = e.message;
+  }
+
+  // DB ping
+  const t0 = Date.now();
+  try {
+    await pool.query('SELECT 1');
+    out.checks.db.ok = true;
+    out.checks.db.ping_ms = Date.now() - t0;
+  } catch (e) {
+    out.checks.db.error = e.message;
+  }
+
+  // canary
+  try {
+    await verifyCanary(pool);
+    out.checks.canary.ok = true;
+  } catch (e) {
+    out.checks.canary.error = e.message;
+  }
+
+  const ready = out.checks.crypto.ok && out.checks.db.ok && out.checks.canary.ok;
+  return res.status(ready ? 200 : 503).json(out);
+});
 
 /**************************************************
  * DB connections

backend/server3.js

@@ -71,20 +71,75 @@ const stripe = new Stripe(process.env.STRIPE_SECRET_KEY, {
   apiVersion: '2024-04-10',
 });
 
- // LIVENESS: process is up; no external deps
+function fprPathFromEnv() {
- app.get('/healthz', (req, res) => res.type('text').send('OK'));
+  const p = (process.env.DEK_PATH || '').trim();
+  return p ? path.join(path.dirname(p), 'dek.fpr') : null;
+}
 
- // READINESS: KMS/DEK/DB are correct
+// 1) Liveness: process is up and event loop responsive
- app.get('/readyz', async (req, res) => {
+app.get('/livez', (_req, res) => res.type('text').send('OK'));
+
+// 2) Readiness: crypto + canary are good
+app.get('/readyz', async (_req, res) => {
   try {
-     await verifyCanary(pool);           // throws if bad
+    await initEncryption();      // load/unlock DEK
-     return res.type('text').send('READY');
+    await verifyCanary(pool);    // DB + decrypt sentinel
+    return res.type('text').send('OK');
   } catch (e) {
     console.error('[READYZ]', e.message);
-     return res.status(503).type('text').send('NOT_READY');
+    return res.status(500).type('text').send('FAIL');
   }
- });
+});
 
+// 3) Health: detailed JSON (you can curl this to “see everything”)
+app.get('/healthz', async (_req, res) => {
+  const out = {
+    service: process.env.npm_package_name || 'server',
+    version: process.env.IMG_TAG || null,
+    uptime_s: Math.floor(process.uptime()),
+    now: new Date().toISOString(),
+    checks: {
+      live: { ok: true },           // if we reached here, process is up
+      crypto: { ok: false, fp: null },
+      db: { ok: false, ping_ms: null },
+      canary: { ok: false }
+    }
+  };
+
+  // crypto / DEK
+  try {
+    await initEncryption();
+    out.checks.crypto.ok = true;
+    const p = fprPathFromEnv();
+    if (p) {
+      try { out.checks.crypto.fp = (await readFile(p, 'utf8')).trim(); }
+      catch { /* fp optional */ }
+    }
+  } catch (e) {
+    out.checks.crypto.error = e.message;
+  }
+
+  // DB ping
+  const t0 = Date.now();
+  try {
+    await pool.query('SELECT 1');
+    out.checks.db.ok = true;
+    out.checks.db.ping_ms = Date.now() - t0;
+  } catch (e) {
+    out.checks.db.error = e.message;
+  }
+
+  // canary
+  try {
+    await verifyCanary(pool);
+    out.checks.canary.ok = true;
+  } catch (e) {
+    out.checks.canary.error = e.message;
+  }
+
+  const ready = out.checks.crypto.ok && out.checks.db.ok && out.checks.canary.ok;
+  return res.status(ready ? 200 : 503).json(out);
+});
 
 function internalFetch(req, urlPath, opts = {}) {
   return fetch(`${API_BASE}${urlPath}`, {

docker-compose.yml

@@ -51,10 +51,11 @@ services:
       - ./user_profile.db:/app/user_profile.db
       - dek-vol:/run/secrets/dev:rw
     healthcheck:
-      test: ["CMD-SHELL", "curl -f http://localhost:${SERVER1_PORT}/healthz || exit 1"]
+      test: ["CMD-SHELL", "curl -fsS http://localhost:${SERVER1_PORT}/livez || exit 1"]
-      interval: 30s
+      interval: 15s
       timeout: 5s
-      retries: 3
+      retries: 5
+      start_period: 25s
 
   # ───────────────────────────── server2 ─────────────────────────────
   server2:
@@ -96,10 +97,11 @@ services:
       - ./user_profile.db:/app/user_profile.db
       - dek-vol:/run/secrets/dev:ro
     healthcheck:
-      test: ["CMD-SHELL", "curl -f http://localhost:${SERVER2_PORT}/healthz || exit 1"]
+      test: ["CMD-SHELL", "curl -fsS http://localhost:${SERVER2_PORT}/livez || exit 1"]
-      interval: 30s
+      interval: 15s
       timeout: 5s
-      retries: 3
+      retries: 5
+      start_period: 25s
 
   # ───────────────────────────── server3 ─────────────────────────────
   server3:
@@ -148,10 +150,11 @@ services:
       - ./user_profile.db:/app/user_profile.db
       - dek-vol:/run/secrets/dev:ro
     healthcheck:
-      test: ["CMD-SHELL", "curl -f http://localhost:${SERVER3_PORT}/healthz || exit 1"]
+      test: ["CMD-SHELL", "curl -fsS http://localhost:${SERVER3_PORT}/livez || exit 1"]
-      interval: 30s
+      interval: 15s
       timeout: 5s
-      retries: 3
+      retries: 5
+      start_period: 25s
 
   # ───────────────────────────── nginx ───────────────────────────────
   nginx: