secureCodeBox · J12934 · Jul 27, 2021 · Jul 20, 2021 · Jul 20, 2021 · Jul 20, 2021
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -10,6 +10,8 @@ env:
 
   GO_VERSION: "1.15"
   PYTHON_VERSION: "3.9"
+  NODE_VERSION: "14"
+  NPM_VERSION: "7"
 
   # ---- Docker Namespace ----
 
@@ -117,7 +119,7 @@ jobs:
         env:
           CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
         with:
-          coverageCommand: npm test -- --ci --colors --coverage
+          coverageCommand: npm test -- --ci --colors --coverage --testPathIgnorePatterns=/integration-tests/
 
   # ---- Build Stage ----
 
@@ -280,6 +282,108 @@ jobs:
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}
 
+# ---- New Makefile based CI Pipeline steps ----
+
+  makefile-scanner:
+    name: "Make Scanners"
+    needs:
+      - sdk
+      - operator
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        unit: ["amass"]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v2
+        name: Setup Node
+        with:
+          node-version: ${{ env. NODE_VERSION }}
+
+      - name: Update NPM
+        run: npm i -g npm@${{ env.NPM_VERSION }}
+
+      - name: Install Dependencies
+        working-directory: ./scanners/${{ matrix.unit }}/
+        run: make install-deps
+
+      - name: Unit Tests
+        working-directory: ./scanners/${{ matrix.unit }}/
+        run: make unit-tests
+
+      - name: Set baseImageTag To commit Hash
+        run: |
+          echo "baseImageTag=sha-$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
+      - name: Docker Meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ${{ env.DOCKER_NAMESPACE }}/parser-${{ matrix.unit }}
+          tag-sha: true
+          tag-semver: |
+            {{ version }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./scanners/${{ matrix.unit }}/parser/
+          file: ./scanners/${{ matrix.unit }}/parser/Dockerfile
+          load: true
+          build-args: |
+            namespace=${{ env.DOCKER_NAMESPACE }}
+            baseImageTag=${{ env.baseImageTag }}
+          platforms: linux/amd64
+          push: ${{ github.event_name == 'release' }}
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
+
+      - name: Export Docker Images
+        working-directory: ./scanners/${{ matrix.unit }}/
+        run: make docker-export
+
+      - name: "Start kind cluster"
+        run: |
+          kind version
+          kind create cluster --image kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6 --wait 3m
+
+      - name: "Inspect kind cluster"
+        run: |
+          kubectl config current-context
+          kubectl get node
+
+      - name: Kind Import Images
+        working-directory: ./scanners/${{ matrix.unit }}/
+        run: make kind-import
+
+      - name: Install Operator
+        working-directory: ./operator
+        run: |
+          make helm-deploy
+
+      - name: Deploy ${{ matrix.unit }}
+        working-directory: ./scanners/${{ matrix.unit }}/
+        run: make deploy
+
+      - name: Deploy Test Dependencies
+        working-directory: ./scanners/${{ matrix.unit }}/
+        run: make deploy-test-deps
+
+      - name: Start Integration Tests
+        working-directory: ./scanners/${{ matrix.unit }}/
+        run: make integration-tests
+
   # ---- Build Stage | Matrix Hooks ----
 
   hooks:
@@ -353,7 +457,6 @@ jobs:
     strategy:
       matrix:
         parser:
-          - amass
           - angularjs-csti-scanner
           - git-repo-scanner
           - gitleaks
@@ -770,17 +873,6 @@ jobs:
           kubectl create deployment --image nginx:alpine nginx --namespace demo-targets
           kubectl expose deployment nginx --port 80 --namespace demo-targets
 
-      # ---- OWASP Amass Integration Tests ----
-
-      - name: "amass Integration Tests"
-        run: |
-          kubectl -n integration-tests delete scans --all
-          helm -n integration-tests install amass ./scanners/amass/ \
-            --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-amass" \
-            --set="parser.image.tag=sha-$(git rev-parse --short HEAD)"
-          cd tests/integration/
-          npx jest --ci --color scanner/amass.test.js
-
       # ---- gitleaks Integration Tests ----
 
       - name: "gitleaks Integration Tests"

diff --git a/operator/Makefile b/operator/Makefile
@@ -2,8 +2,17 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-# Image URL to use all building/pushing image targets
-IMG ?= securecodebox/operator:latest
+IMG_NS ?= securecodebox
+
+# Image URL to use all building/pushing image targets for the operator
+OPERATOR_IMG ?= operator
+
+# Image URL to use all building/pushing image targets for the lurker
+LURKER_IMG ?= lurker
+
+# Tag used for the images
+IMG_TAG ?= sha-$$(git rev-parse --short HEAD)
+
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"
 
@@ -67,21 +76,48 @@ run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./main.go
 
 docker-build: test ## Build docker image with the manager.
-	docker build -t ${IMG} .
+	@echo ".: ⚙️ Build Container Images"
+	docker build -t $(IMG_NS)/${OPERATOR_IMG}:${IMG_TAG} .
+	cd ../lurker && docker build -t $(IMG_NS)/$(LURKER_IMG):$(IMG_TAG) .
 
 docker-push: ## Push docker image with the manager.
-	docker push ${IMG}
+	docker push $(IMG_NS)/${OPERATOR_IMG}:${IMG_TAG}
+	docker push $(IMG_NS)/$(LURKER_IMG):$(IMG_TAG)
+
+docker-export:
+	@echo ".: 💾 Export Container Images"
+	docker save $(IMG_NS)/$(OPERATOR_IMG):$(IMG_TAG) > $(OPERATOR_IMG).tar
+	docker save $(IMG_NS)/$(LURKER_IMG):$(IMG_TAG) > $(LURKER_IMG).tar
 
 ##@ Deployment
 
+kind-import:
+	@echo ".: 💾 Importing the image archive to local kind cluster."
+	kind load image-archive ./$(OPERATOR_IMG).tar
+	kind load image-archive ./$(LURKER_IMG).tar
+
+helm-deploy:
+	@echo ".: ⚙️ Deploying Operator with the Image tag '$(IMG_TAG)' into kind."
+	# If not exists create namespace where the tests will be executed
+	kubectl create namespace integration-tests --dry-run=client -o yaml | kubectl apply -f -
+	# If not exists create secureCodeBox operator namespace
+	kubectl create namespace securecodebox-system --dry-run=client -o yaml | kubectl apply -f -
+	helm -n securecodebox-system upgrade --install securecodebox-operator ./ --wait \
+    --set="image.repository=docker.io/$(IMG_NS)/$(OPERATOR_IMG)" \
+    --set="image.tag=$(IMG_TAG)" \
+		--set="image.pullPolicy=IfNotPresent" \
+    --set="lurker.image.repository=docker.io/$(IMG_NS)/$(LURKER_IMG)" \
+    --set="lurker.image.tag=$(IMG_TAG)" \
+		--set="lurker.pullPolicy=IfNotPresent"
+
 install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/crd | kubectl apply -f -
 
 uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/crd | kubectl delete -f -
 
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${OPERATOR_IMG}
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
 
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.

diff --git a/scanners/amass/.gitignore b/scanners/amass/.gitignore
@@ -0,0 +1 @@
+*.tar
diff --git a/scanners/amass/.helmignore b/scanners/amass/.helmignore
@@ -1,9 +1,36 @@
 # SPDX-FileCopyrightText: 2020 iteratec GmbH
 #
 # SPDX-License-Identifier: Apache-2.0
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
 .DS_Store
-
-parser/
-scanner/
-examples/
-docs/
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
+# Node.js files
+node_modules/*
+package.json
+package-lock.json
+src/*
+config/*
+Dockerfile
+.dockerignore
+*.tar
+parser/*
+scanner/*
diff --git a/scanners/amass/Makefile b/scanners/amass/Makefile
@@ -0,0 +1,112 @@
+#!/usr/bin/make -f
+#
+# SPDX-FileCopyrightText: 2021 iteratec GmbH
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# This Makefile is intended to be used for developement and testing only.
+# For using this scanner/hook in production please use the helm chart.
+# See: <https://docs.securecodebox.io/docs/getting-started/installation>
+#
+# This Makefile expects some additional software to be installed:
+# - git
+# - node + npm
+# - docker
+# - kind
+# - kubectl
+# - helm
+
+# Thx to https://stackoverflow.com/questions/5618615/check-if-a-program-exists-from-a-makefile
+EXECUTABLES = make docker kind git node npm npx kubectl helm
+K := $(foreach exec,$(EXECUTABLES),\
+        $(if $(shell which $(exec)),some string,$(error "ERROR: The prerequisites are not met to execute this makefile! No '$(exec)' found in your PATH")))
+
+
+# Variables you might want to override:
+#
+# IMG_NS:				Defines the namespace under which the images are build.
+#						For `securecodebox/scanner-nmap` `securecodebox` is the namespace
+#						Defaults to `securecodebox`
+#
+# BASE_IMG_TAG:			Defines the tag of the base image used to build this scanner/hook
+#
+# IMG_TAG:				Tag used to tag the newly created image. Defaults to the shortend commit hash
+#						prefixed with `sha-` e.g. `sha-ef8de4b7`
+#
+# JEST_VERSION  		Defines the jest version used for executing the tests. Defaults to latest
+#
+# Examples:
+# 	make all IMG_TAG=main
+# 	make deploy IMG_TAG=$(git rev-parse --short HEAD)
+# 	make integration-tests
+#
+
+SHELL = /bin/sh
+
+IMG_NS ?= securecodebox
+GIT_TAG ?= $$(git rev-parse --short HEAD)
+BASE_IMG_TAG ?= latest
+IMG_TAG ?= "sha-$(GIT_TAG)"
+JEST_VERSION ?= latest
+
+scanner = amass
+scanner-prefix = scanner
+parser-prefix = parser
+
+
+build: | install-deps docker-build
+
+test: | unit-tests docker-export kind-import deploy deploy-test-deps integration-tests
+
+all: | clean install-deps unit-tests docker-build docker-export kind-import deploy deploy-test-deps integration-tests
+
+.PHONY: unit-tests install-deps docker-build docker-export kind-import deploy deploy-test-deps integration-tests all build test
+
+unit-tests:
+	@echo ".: 🧪 Starting unit-tests for '$(scanner)' parser  with 'jest@$(JEST_VERSION)'."
+	cd parser && npx --yes --package jest@$(JEST_VERSION) jest --ci --colors --coverage .
+
+install-deps:
+	@echo ".: ⚙️ Installing all scanner specific dependencies."
+	cd ./.. && npm ci
+	cd ../../parser-sdk/nodejs && npm ci
+	cd ./parser/ && npm ci
+
+docker-build:
+	@echo ".: ⚙️ Build With BASE_IMG_TAG: '$(BASE_IMG_TAG)'."
+	docker build --build-arg=baseImageTag=$(BASE_IMG_TAG) --build-arg=namespace=$(IMG_NS) -t $(IMG_NS)/$(parser-prefix)-$(scanner):$(IMG_TAG) -f ./parser/Dockerfile ./parser
+
+docker-export:
+	@echo ".: ⚙️ Saving new docker image archive to '$(parser-prefix)-$(scanner).tar'."
+	docker save $(IMG_NS)/$(parser-prefix)-$(scanner):$(IMG_TAG) -o $(parser-prefix)-$(scanner).tar
+
+kind-import:
+	@echo ".: 💾 Importing the image archive '$(parser-prefix)-$(scanner).tar' to local kind cluster."
+	kind load image-archive ./$(parser-prefix)-$(scanner).tar
+
+deploy:
+	@echo ".: 💾 Deploying '$(scanner)' scanner HelmChart with the docker tag '$(IMG_TAG)' into kind namespace 'integration-tests'."
+	helm -n integration-tests upgrade --install $(scanner) ./ --wait \
+		--set="parser.image.repository=docker.io/$(IMG_NS)/$(parser-prefix)-$(scanner)" \
+		--set="parser.image.tag=$(IMG_TAG)"
+
+deploy-test-deps:
+
+install-integration-test-deps:
+
+integration-tests:
+	@echo ".: 🩺 Starting integration test in kind namespace 'integration-tests'."
+	kubectl -n integration-tests delete scans --all
+	cd ../../tests/integration/ && npm ci
+	npx --yes --package jest@$(JEST_VERSION) jest --ci --colors --coverage ./integration-tests
+
+clean:
+	@echo ".: 🧹 Cleaning up all generated files."
+	rm -f ./$(parser-prefix)-$(scanner).tar
+	rm -rf ./parser/node_modules
+	rm -rf ./parser/coverage
+	rm -rf ./integration-tests/node_modules
+	rm -rf ./integration-tests/coverage
+	rm -rf ../node_modules
+	rm -rf ../coverage
diff --git a/tests/integration/scanner/amass.test.js → ...ers/amass/integration-tests/amass.test.js b/tests/integration/scanner/amass.test.js → ...ers/amass/integration-tests/amass.test.js
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-const { scan } = require("../helpers");
+const { scan } = require("../../../tests/integration/helpers.js");
 
 jest.retryTimes(3);
 

diff --git a/scanners/amass/integration-tests/jest.config.json b/scanners/amass/integration-tests/jest.config.json
@@ -0,0 +1 @@
+{}
diff --git a/scanners/amass/parser/jest.config.json b/scanners/amass/parser/jest.config.json
@@ -0,0 +1 @@
+{}