Deploy e2e compose stack with health check and validate it restarts failed container

Signed-off-by: Guillaume Tardif <[email protected]>

tests/aci-e2e/e2e-aci_test.go

@@ -591,6 +591,41 @@ func TestUpSecretsResources(t *testing.T) {
 		assert.Equal(t, web2Inspect.HostConfig.CPUReservation, 0.5)
 		assert.Equal(t, web2Inspect.HostConfig.MemoryReservation, uint64(751619276))
 	})
+
+	t.Run("check healthchecks inspect", func(t *testing.T) {
+		assert.Equal(t, web1Inspect.Healthcheck.Disable, false)
+		assert.Equal(t, time.Duration(web1Inspect.Healthcheck.Interval), 5*time.Second)
+		assert.DeepEqual(t, web1Inspect.Healthcheck.Test, []string{"curl", "-f", "http://localhost:80/healthz"})
+
+		assert.Equal(t, web2Inspect.Healthcheck.Disable, true)
+	})
+
+	t.Run("healthcheck restart failed app", func(t *testing.T) {
+		endpoint := fmt.Sprintf("http://%s:%d", web1Inspect.Ports[0].HostIP, web1Inspect.Ports[0].HostPort)
+		HTTPGetWithRetry(t, endpoint+"/failtestserver", http.StatusOK, 3*time.Second, 3*time.Second)
+
+		logs := c.RunDockerCmd("logs", web1).Combined()
+		assert.Assert(t, strings.Contains(logs, "GET /healthz"))
+		assert.Assert(t, strings.Contains(logs, "GET /failtestserver"))
+		assert.Assert(t, strings.Contains(logs, "Server failing"))
+
+		checkLogsReset := func(logt poll.LogT) poll.Result {
+			res := c.RunDockerOrExitError("logs", web1)
+			if res.ExitCode == 0 &&
+				!strings.Contains(res.Combined(), "GET /failtestserver") &&
+				strings.Contains(res.Combined(), "Listening on port 80") &&
+				strings.Contains(res.Combined(), "GET /healthz") {
+				return poll.Success()
+			}
+			return poll.Continue("Logs not reset by healcheck restart\n" + res.Combined())
+		}
+
+		poll.WaitOn(t, checkLogsReset, poll.WithDelay(5*time.Second), poll.WithTimeout(90*time.Second))
+
+		res := c.RunDockerCmd("inspect", web1)
+		web1Inspect, err = ParseContainerInspect(res.Stdout())
+		assert.Equal(t, web1Inspect.Status, "Running")
+	})
 }
 
 func TestUpUpdate(t *testing.T) {

tests/composefiles/aci_secrets_resources/compose.yml

@@ -9,6 +9,8 @@ services:
         target: mytarget1
       - mysecret2
     deploy:
+      restart_policy:
+        condition: on-failure
       resources:
         limits:
           cpus: '0.7'
@@ -16,6 +18,9 @@ services:
         reservations:
           cpus: '0.5'
           memory: 0.5G
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:80/healthz"]
+      interval: 5s
 
   web2:
     build: ./web
@@ -25,6 +30,8 @@ services:
     environment:
       - PORT=8080
     deploy:
+      restart_policy:
+        condition: on-failure
       resources:
         reservations:
           cpus: '0.5'

tests/composefiles/aci_secrets_resources/web/Dockerfile

@@ -21,5 +21,6 @@ RUN --mount=type=cache,target=/go/pkg/mod \
     CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o server main.go
 
 FROM alpine
+RUN apk --no-cache add curl
 COPY --from=build /go/server /
 CMD /server "${PORT:-80}" "${DIR:-/run/secrets}"

tests/composefiles/aci_secrets_resources/web/main.go

@@ -17,11 +17,49 @@
 package main
 
 import (
-	"log"
+	"fmt"
 	"net/http"
 	"os"
 )
 
 func main() {
-	log.Fatal(http.ListenAndServe(":"+os.Args[1], http.FileServer(http.Dir(os.Args[2]))))
+	if len(os.Args) < 2 {
+		fmt.Fprintln(os.Stderr, "Usage: web PORT FOLDER")
+		os.Exit(1)
+	}
+
+	http.HandleFunc("/failtestserver", log(fail))
+	http.HandleFunc("/healthz", log(healthz))
+	dir := os.Args[2]
+	fileServer := http.FileServer(http.Dir(dir))
+	http.HandleFunc("/", log(fileServer.ServeHTTP))
+
+	port := os.Args[1]
+	fmt.Println("Listening on port " + port)
+	err := http.ListenAndServe(":"+port, nil)
+	if err != nil {
+		fmt.Printf("Error while starting server: %v", err)
+	}
+}
+
+var healthy bool = true
+
+func fail(w http.ResponseWriter, req *http.Request) {
+	healthy = false
+	fmt.Println("Server failing")
+}
+
+func healthz(w http.ResponseWriter, r *http.Request) {
+	if !healthy {
+		fmt.Println("unhealthy")
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+}
+
+func log(handler func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		fmt.Println(r.Method, r.URL.Path, r.RemoteAddr, r.UserAgent())
+		handler(w, r)
+	}
 }