Test current PR build and add local registry in KinD

.github/workflows/build.yaml

@@ -3,37 +3,56 @@ name: build
 on:
   push:
     branches: [ '*' ]
-  pull_request:
-    branches: [ '*' ]
 
 jobs:
-  build:
+  build-test:
     strategy:
       matrix:
         go-version: [1.13.x]
         os: [ubuntu-latest]
+        operator: [0, 1]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@master
       - name: Install Go
         uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.go-version }}
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
+      - name: Kubernetes KinD Cluster
+        uses: container-tools/kind-action@v1
       - name: Build x86_64 container into library
-        uses: docker/build-push-action@v2
+        run: |
+          docker build --platform=linux/amd64 --tag=kind-registry:5000/openfaas/faas-netes:${{ github.sha }} .
+      - name: Push image
+        run: |
+          docker push kind-registry:5000/openfaas/faas-netes:${{ github.sha }}
+      - name: get tools
+        run: ./contrib/get_tools.sh
+      - name: lint chart
+        run: ./contrib/lint_chart.sh
+      - name: deploy function
+        run: OPERATOR=${{ matrix.operator }} FAASNETES_IMAGE=kind-registry:5000/openfaas/faas-netes:${{ github.sha }} ./contrib/deploy.sh
+      - name: run function
+        run: OPERATOR=${{ matrix.operator }} ./contrib/run_function.sh
+      - name: load test
+        if: ${{ matrix.operator == 1 }}
+        run: ./contrib/load_test.sh
+  build-multiarch:
+    strategy:
+      matrix:
+        go-version: [1.13.x]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@master
+      - name: Install Go
+        uses: actions/setup-go@v2
         with:
-          context: .
-          file: ./Dockerfile
-          outputs: "type=docker,push=false"
-          platforms: linux/amd64
-          tags: |
-            ghcr.io/openfaas/faas-netes:${{ github.sha }}
-
+          go-version: ${{ matrix.go-version }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - uses: docker/setup-buildx-action@v1
+        id: buildx
       - name: Build multi-arch containers for validation only
         uses: docker/build-push-action@v2
         with:
@@ -43,31 +62,5 @@ jobs:
           platforms: linux/amd64,linux/arm/v7,linux/arm64
           tags: |
             ghcr.io/openfaas/faas-netes:${{ github.sha }}
- # Todo - load the image into Kind before running tests
- # otherwise, this just tests the chart with images that 
- # have already been built and pushed in an earlier job
-      - name: get tools
-        run: ./contrib/get_tools.sh
-      - name: lint chart
-        run: ./contrib/lint_chart.sh
-      - name: create cluster
-        run: ./contrib/create_cluster.sh
-      - name: deploy function
-        run: OPERATOR=0 ./contrib/deploy.sh
-      - name: run function
-        run: OPERATOR=0 ./contrib/run_function.sh
-      - name: stop dev cluster
-        run: ./contrib/stop_dev.sh
-      - name: wait 10 seconds
-        run: sleep 10
-      - name: create cluster
-        run: ./contrib/create_cluster.sh
-      - name: deploy function
-        run: OPERATOR=1 ./contrib/deploy.sh
-      - name: run function
-        run: OPERATOR=1 ./contrib/run_function.sh
-      - name: stop dev cluster
-        run: ./contrib/stop_dev.sh
-
 # The certifier should also be run here
 #

Makefile

@@ -1,6 +1,7 @@
 .PHONY: build local push namespaces install charts start-kind stop-kind build-buildx render-charts
 TAG?=latest
 OWNER?=openfaas
+REGISTRY?=ghcr.io
 export DOCKER_CLI_EXPERIMENTAL=enabled
 
 all: build-docker
@@ -10,15 +11,15 @@ local:
 
 build-docker:
 	docker build \
-	-t ghcr.io/$(OWNER)/faas-netes:$(TAG) .
+	-t $(REGISTRY)/$(OWNER)/faas-netes:$(TAG) .
 
 .PHONY: build-buildx
 build-buildx:
 	@docker buildx create --use --name=multiarch --node=multiarch && \
 	docker buildx build \
 		--output "type=docker,push=false" \
 		--platform linux/amd64 \
-		--tag ghcr.io/$(OWNER)/faas-netes:$(TAG) \
+		--tag $(REGISTRY)/$(OWNER)/faas-netes:$(TAG) \
 		.
 
 .PHONY: build-buildx-all
@@ -27,21 +28,21 @@ build-buildx-all:
 	docker buildx build \
 		--platform linux/amd64,linux/arm/v7,linux/arm64 \
 		--output "type=image,push=false" \
-		--tag ghcr.io/$(OWNER)/faas-netes:$(TAG) \
+		--tag $(REGISTRY)/$(OWNER)/faas-netes:$(TAG) \
 		.
 
 .PHONY: publish-buildx-all
 publish-buildx-all:
-	@echo  ghcr.io/$(OWNER)/faas-netes:$(TAG) && \
+	@echo  $(REGISTRY)/$(OWNER)/faas-netes:$(TAG) && \
 	docker buildx create --use --name=multiarch --node=multiarch && \
 	docker buildx build \
 		--platform linux/amd64,linux/arm/v7,linux/arm64 \
 		--push=true \
-		--tag ghcr.io/$(OWNER)/faas-netes:$(TAG) \
+		--tag $(REGISTRY)/$(OWNER)/faas-netes:$(TAG) \
 		.
 
 push:
-	docker push ghcr.io/$(OWNER)/faas-netes:$(TAG)
+	docker push $(REGISTRY)/$(OWNER)/faas-netes:$(TAG)
 
 namespaces:
 	kubectl apply -f namespaces.yml

chart/openfaas/templates/prometheus-rbac.yaml

@@ -83,6 +83,7 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
   name: {{ .Release.Name }}-prometheus
+  namespace: {{ .Release.Namespace | quote }}
   labels:
     app: {{ template "openfaas.name" . }}
     chart: {{ .Chart.Name }}-{{ .Chart.Version }}

contrib/create_cluster.sh

@@ -1,11 +1,40 @@
 #!/usr/bin/env bash
-
+set -o errexit
 DEVENV=${OF_DEV_ENV:-kind}
 KUBE_VERSION=v1.18.8
+REG_NAME=kind-registry
+REG_PORT=5000
 
+./contrib/create_local_registry.sh ${REG_NAME} ${REG_PORT}
 echo ">>> Creating Kubernetes ${KUBE_VERSION} cluster ${DEVENV}"
 
-kind create cluster --wait 5m --image kindest/node:${KUBE_VERSION} --name "$DEVENV" -v 1
+# create a cluster with the local registry enabled in containerd
+cat <<EOF | kind create cluster \
+--wait 5m --image kindest/node:${KUBE_VERSION} --name "$DEVENV" -v 1 --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."${REG_NAME}:${REG_PORT}"]
+    endpoint = ["http://${REG_NAME}:${REG_PORT}"]
+EOF
+
+
+# connect the registry to the cluster network
+# (the network may already be connected)
+containers=$(docker network inspect kind -f "{{range .Containers}}{{.Name}} {{end}}")
+needs_connect="true"
+for c in $containers; do
+  if [ "$c" = "${reg_name}" ]; then
+    needs_connect="false"
+  fi
+done
+if [ "${needs_connect}" = "true" ]; then               
+  docker network connect "kind" "${REG_NAME}" || true
+else
+	echo ">>> Kind network already connected to local registry"
+fi
+
 
 echo ">>> Waiting for CoreDNS"
 kubectl --context "kind-$DEVENV" -n kube-system rollout status deployment/coredns

contrib/create_local_registry.sh

@@ -0,0 +1,9 @@
+# create registry container unless it already exists
+REG_NAME=${1:-kind-registry}
+REG_PORT=${2:-5000}
+running="$(docker inspect -f '{{.State.Running}}' "${REG_NAME}" 2>/dev/null || false)"
+if [ "${running}" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "127.0.0.1:${REG_PORT}:5000" --name "${REG_NAME}" \
+    registry:2
+fi

contrib/deploy.sh

@@ -4,6 +4,8 @@ set -e
 
 DEVENV=${OF_DEV_ENV:-kind}
 OPERATOR=${OPERATOR:-0}
+FAASNETES_IMAGE=${FAASNETES_IMAGE:-ghcr.io/openfaas/faas-netes:0.12.18}
+
 
 echo "Applying namespaces"
 kubectl --context "kind-$DEVENV" apply -f ./namespaces.yml
@@ -17,35 +19,42 @@ if [ -x "$(command -v $sha_cmd)" ]; then
     sha_cmd="shasum"
 fi
 
-PASSWORD=$(head -c 16 /dev/urandom| $sha_cmd | cut -d " " -f 1)
-echo -n $PASSWORD > password.txt
 
+# Only create a new password and secret if it does not already exist in the cluster
+# This makes the script idempotent.
+kubectl get secret basic-auth -n openfaas --context "kind-$DEVENV" > /dev/null || \
+(PASSWORD=$(head -c 16 /dev/urandom| $sha_cmd | cut -d " " -f 1) && \
+echo -n $PASSWORD > password.txt && \
 kubectl --context "kind-$DEVENV" -n openfaas create secret generic basic-auth \
 --from-literal=basic-auth-user=admin \
---from-literal=basic-auth-password="$PASSWORD"
+--from-literal=basic-auth-password="$PASSWORD")
 
 CREATE_OPERATOR=false
 if [ "${OPERATOR}" == "1" ]; then
     CREATE_OPERATOR="true"
 fi
 
 echo "Waiting for helm install to complete."
+echo "Using faasnetes image $FAASNETES_IMAGE"
 
-helm upgrade \
-    --kube-context "kind-$DEVENV" \
-    --install \
+helm template \
     openfaas \
     ./chart/openfaas \
     --namespace openfaas  \
     --set basic_auth=true \
+    --set faasnetes.image=$FAASNETES_IMAGE \
     --set functionNamespace=openfaas-fn \
-    --set operator.create=$CREATE_OPERATOR
+    --set queueWorker.image=kind-registry:5000/cognitedata/queue-worker:dda07bf-amd64 \
+    --set gateway.image=openfaas/gateway:0.18.18 \
+    --debug > kustomize/base/base.yaml
+kustomize build kustomize/local | kubectl apply -f -
+
 
 if [ "${OPERATOR}" == "1" ]; then
 
     kubectl --context "kind-$DEVENV" patch -n openfaas deploy/gateway \
       -p='[{"op": "add", "path": "/spec/template/spec/containers/1/command", "value": ["./faas-netes", "-operator=true"]} ]' --type=json
 fi
 
-kubectl --context "kind-$DEVENV" rollout status deploy/prometheus -n openfaas
-kubectl --context "kind-$DEVENV" rollout status deploy/gateway -n openfaas
+kubectl --context "kind-$DEVENV" rollout status deploy/prometheus -n openfaas --timeout=2m
+kubectl --context "kind-$DEVENV" rollout status deploy/gateway -n openfaas --timeout=4m

contrib/get_tools.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-KIND_VERSION="v0.9.0"
+KIND_VERSION="v0.10.0"
 # Causes a validation failure when linting due to CRDs moving to v1
 # HELM_VERSION="v3.4.0"
 HELM_VERSION="v3.0.3"
@@ -12,7 +12,9 @@ ARKADE_VERSION="0.6.21"
 echo "Downloading arkade"
 
 curl -SLs https://github.com/alexellis/arkade/releases/download/$ARKADE_VERSION/arkade > arkade
+curl -SLs https://hey-release.s3.us-east-2.amazonaws.com/hey_linux_amd64 > hey
 chmod +x ./arkade
+chmod +x ./hey
 
 
 if [[ "$1" ]]; then
@@ -25,3 +27,4 @@ fi
 ./arkade get faas-cli
 
 sudo mv $HOME/.arkade/bin/* /usr/local/bin/
+sudo mv ./hey /usr/local/bin/

contrib/load_test.sh

@@ -0,0 +1,11 @@
+# Load test with scaling
+
+FXN_NAME=echo
+export OPENFAAS_URL=http://127.0.0.1:31112
+hey -z=5s -q 5 -c 2 -m POST -d=Test $OPENFAAS_URL/function/$FXN_NAME # All should give 200
+sleep 5
+kubectl scale --replicas=0 deploy/$FXN_NAME -n openfaas-fn
+kubectl scale --replicas=1 deploy/$FXN_NAME -n openfaas-fn
+kubectl --context "kind-kind" rollout status deploy/$FXN_NAME -n openfaas-fn --timeout=4m
+hey -z=5s -q 5 -c 2 -m POST -d=Test $OPENFAAS_URL/async-function/$FXN_NAME
+kubectl scale --replicas=0 deploy/$FXN_NAME -n openfaas-fn 

contrib/load_test_sleep.sh

@@ -0,0 +1,5 @@
+# Load test with scaling
+DEVENV=${DEVENV:-kind}
+export OPENFAAS_URL=http://127.0.0.1:31112
+hey -z=5s -q 5 -c 2 -m POST -d=Test $OPENFAAS_URL/async-function/sleep
+hey -z=5s -q 5 -c 2 -m POST -d=Test $OPENFAAS_URL/function/sleep

contrib/restart_port_forward.sh

@@ -3,7 +3,7 @@
 DEVENV=${OF_DEV_ENV:-kind}
 
 if [ -f "of_${DEVENV}_portforward.pid" ]; then
-    kill $(<of_${DEVENV}_portforward.pid)
+    kill $(<of_${DEVENV}_portforward.pid) || echo "No port-forward process running"
 fi
 
 # quietly start portforward and put it in the background, it will not

contrib/run_function.sh

@@ -13,7 +13,8 @@ then
 fi
 
 if [ -f "of_${DEVENV}_portforward.pid" ]; then
-    kill $(<of_${DEVENV}_portforward.pid)
+    # If the kill fails, there is no process running and the file should be removed
+    kill $(<of_${DEVENV}_portforward.pid) || rm of_${DEVENV}_portforward.pid
 fi
 
 # quietly start portforward and put it in the background, it will not
@@ -33,30 +34,31 @@ cat ./password.txt | faas-cli login --username admin --password-stdin
 faas-cli deploy --image=functions/alpine:latest --fprocess=cat --name "echo"
 
 # Call echo function
-for i in {1..180};
-do
+while [[ i -lt 180 && $Ready != "Ready" ]]; do
     Ready="$(faas-cli describe echo | awk '{ if($1 ~ /Status:/) print $2 }')"
     if [[ $Ready == "Ready" ]];
     then
-        exit 0
+        echo "Success: echo function is ready"
     fi
     sleep 1
+    i=$((i + 1))
 done
 
 # Apply a CRD to test the operator
 
 if [ "${OPERATOR}" == "1" ]; then
-    kubectl --context "kind-$DEVENV" apply ./alpine-fn.yaml
-
+    echo "Testing openfaas operator"
+    kubectl --context "kind-$DEVENV" apply -f ./contrib/alpine-fn.yaml
     for i in {1..180};
     do
         Ready="$(faas-cli describe nodeinfo | awk '{ if($1 ~ /Status:/) print $2 }')"
         if [[ $Ready == "Ready" ]];
         then
+            echo "Success: nodeinfo function is ready"
             exit 0
         fi
         sleep 1
     done
 fi
 
-exit 1
+exit 0

contrib/scale_to_zero_test.sh

@@ -0,0 +1,16 @@
+# Load test with scaling
+
+export OPENFAAS_URL=http://127.0.0.1:31112
+# Start with a fresh slate to emulate scale to zero
+kubectl scale --replicas=0 deploy/echo -n openfaas-fn
+sleep 11
+
+# Start request batch #1
+curl -X POST -d Test $OPENFAAS_URL/function/echo # scale to 1
+hey -z=2s -q 5 -c 2 -m POST -d Test $OPENFAAS_URL/function/echo
+sleep 5 # Necessary to not get 404
+# Scale back to zero
+kubectl scale --replicas=0 deploy/echo -n openfaas-fn
+# sleep 5
+# Start request batch #2
+hey -z=2s -q 5 -c 2 -m POST -d Test $OPENFAAS_URL/function/echo # gives 404