From 34ef3fa78b395b9149abf7b40ba00b9069dca278 Mon Sep 17 00:00:00 2001 From: Yannik Fuhrmeister Date: Tue, 20 Jul 2021 08:11:20 +0200 Subject: [PATCH 1/9] Add Makefile For Amass The basic structure of this Makefile should be used for all other scanners and hooks Signed-off-by: Yannik Fuhrmeister --- scanners/amass/Makefile | 111 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 scanners/amass/Makefile diff --git a/scanners/amass/Makefile b/scanners/amass/Makefile new file mode 100644 index 0000000000..bfcf7e82c0 --- /dev/null +++ b/scanners/amass/Makefile @@ -0,0 +1,111 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# +# +# This Makefile is intended to be used for developement and testing only. +# For using this scanner/hook in production please use the helm chart. +# See: +# +# This Makefile expects some additional software to be installed: +# - git +# - node + npm +# - docker +# - kind +# - kubectl +# - helm + +# Thx to https://stackoverflow.com/questions/5618615/check-if-a-program-exists-from-a-makefile +EXECUTABLES = make docker kind git node npm npx kubectl helm +K := $(foreach exec,$(EXECUTABLES),\ + $(if $(shell which $(exec)),some string,$(error "ERROR: The prerequisites are not met to execute this makefile! No '$(exec)' found in your PATH"))) + + +# Variables you might want to override: +# +# IMG_NS: Defines the namespace under which the images are build. +# For `securecodebox/scanner-nmap` `securecodebox` is the namespace +# Defaults to `securecodebox` +# +# BASE_IMG_TAG: Defines the tag of the base image used to build this scanner/hook +# +# IMG_TAG: Tag used to tag the newly created image. Defaults to the shortend commit hash +# prefixed with `sha-` e.g. `sha-ef8de4b7` +# +# JEST_VERSION Defines the jest version used for executing the tests. Defaults to latest +# +# Examples: +# make all IMG_TAG=main +# make deploy IMG_TAG=$(git rev-parse --short HEAD) +# make integration-tests +# + +SHELL = /bin/sh + +IMG_NS ?= securecodebox +GIT_TAG ?= $$(git rev-parse --short HEAD) +BASE_IMG_TAG ?= latest +IMG_TAG ?= "sha-$(GIT_TAG)" +JEST_VERSION ?= latest + +scanner = amass +scanner-prefix = scanner +parser-prefix = parser + + +build: | install-deps docker-build + +test: | unit-tests docker-export kind-import deploy-prerequisites deploy deploy-test-deps integration-tests + +all: | clean install-deps unit-tests docker-build docker-export kind-import deploy-prerequisites deploy deploy-test-deps integration-tests + +.PHONY: unit-tests install-deps docker-build docker-export kind-import deploy-prerequisites deploy deploy-test-deps integration-tests all build test + +unit-tests: + @echo ".: 🧪 Starting unit-tests for '$(scanner)' parser with 'jest@$(JEST_VERSION)'." + cd parser && npx --yes --package jest@$(JEST_VERSION) jest --ci --colors --coverage . + +install-deps: + @echo ".: ⚙️ Installing all scanner specific dependencies." + cd ./.. && npm ci + cd ./parser/ && npm ci + +docker-build: + @echo ".: ⚙️ Build With BASE_IMG_TAG: '$(BASE_IMG_TAG)'." + docker build --build-arg=baseImageTag=$(BASE_IMG_TAG) --build-arg=namespace=$(IMG_NS) -t $(IMG_NS)/$(parser-prefix)-$(scanner):$(IMG_TAG) -f ./parser/Dockerfile ./parser + +docker-export: + @echo ".: ⚙️ Saving new docker image archive to '$(parser-prefix)-$(scanner).tar'." + docker save $(IMG_NS)/$(parser-prefix)-$(scanner):$(IMG_TAG) -o $(parser-prefix)-$(scanner).tar + +kind-import: + @echo ".: 💾 Importing the image archive '$(parser-prefix)-$(scanner).tar' to local kind cluster." + kind load image-archive ./$(parser-prefix)-$(scanner).tar + +deploy: + @echo ".: 💾 Deploying '$(scanner)' scanner HelmChart with the docker tag '$(IMG_TAG)' into kind namespace 'integration-tests'." + helm -n integration-tests upgrade --install $(scanner) ./ --wait \ + --set="parser.image.repository=docker.io/$(IMG_NS)/$(parser-prefix)-$(scanner)" \ + --set="parser.image.tag=$(IMG_TAG)" + +deploy-test-deps: + +install-integration-test-deps: + +integration-tests: + @echo ".: 🩺 Starting integration test in kind namespace 'integration-tests'." + kubectl -n integration-tests delete scans --all + cd ../../tests/integration/ && npm ci + npx --yes --package jest@$(JEST_VERSION) jest --ci --colors --coverage ./integration-tests + +clean: + @echo ".: 🧹 Cleaning up all generated files." + rm -f ./$(parser-prefix)-$(scanner).tar + rm -rf ./parser/node_modules + rm -rf ./parser/coverage + rm -rf ./integration-tests/node_modules + rm -rf ./integration-tests/coverage + rm -rf ../node_modules + rm -rf ../coverage From 19f9c4d0f47405497172a8e15657fe94e4fdf270 Mon Sep 17 00:00:00 2001 From: Yannik Fuhrmeister Date: Tue, 20 Jul 2021 08:15:11 +0200 Subject: [PATCH 2/9] Move Amass Integration Tests Into Amass I had to add an empty `jest.config.js` files some scanners and hooks require specific configurations (e.g. for typescript) Signed-off-by: Yannik Fuhrmeister --- .../scanner => scanners/amass/integration-tests}/amass.test.js | 2 +- scanners/amass/integration-tests/jest.config.json | 1 + scanners/amass/parser/jest.config.json | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) rename {tests/integration/scanner => scanners/amass/integration-tests}/amass.test.js (86%) create mode 100644 scanners/amass/integration-tests/jest.config.json create mode 100644 scanners/amass/parser/jest.config.json diff --git a/tests/integration/scanner/amass.test.js b/scanners/amass/integration-tests/amass.test.js similarity index 86% rename from tests/integration/scanner/amass.test.js rename to scanners/amass/integration-tests/amass.test.js index 7f1c5b5914..d23fc2d343 100644 --- a/tests/integration/scanner/amass.test.js +++ b/scanners/amass/integration-tests/amass.test.js @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -const { scan } = require("../helpers"); +const { scan } = require("../../../tests/integration/helpers.js"); jest.retryTimes(3); diff --git a/scanners/amass/integration-tests/jest.config.json b/scanners/amass/integration-tests/jest.config.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/scanners/amass/integration-tests/jest.config.json @@ -0,0 +1 @@ +{} diff --git a/scanners/amass/parser/jest.config.json b/scanners/amass/parser/jest.config.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/scanners/amass/parser/jest.config.json @@ -0,0 +1 @@ +{} From aa35eb88f2cdb2d4f4bfcc9ce503056db7cb8eb8 Mon Sep 17 00:00:00 2001 From: Yannik Fuhrmeister Date: Tue, 20 Jul 2021 08:21:13 +0200 Subject: [PATCH 3/9] Exclude `*.tar` Files I excluded all `.tar` files that they don't end up in the docker context or the repo Signed-off-by: Yannik Fuhrmeister --- scanners/amass/.gitignore | 1 + scanners/amass/.helmignore | 37 ++++++++++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 scanners/amass/.gitignore diff --git a/scanners/amass/.gitignore b/scanners/amass/.gitignore new file mode 100644 index 0000000000..d874ad67cc --- /dev/null +++ b/scanners/amass/.gitignore @@ -0,0 +1 @@ +*.tar diff --git a/scanners/amass/.helmignore b/scanners/amass/.helmignore index 5df91458a9..83c09e25cb 100644 --- a/scanners/amass/.helmignore +++ b/scanners/amass/.helmignore @@ -1,9 +1,36 @@ # SPDX-FileCopyrightText: 2020 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* From f384220e0fbc0318c2143ae960430cddf51fb479 Mon Sep 17 00:00:00 2001 From: Yannik Fuhrmeister Date: Tue, 20 Jul 2021 09:36:22 +0200 Subject: [PATCH 4/9] Add Image Build And Helm Deployment To Makefile To simplify building and deploying images for the operator locally I decided to add some targets to the Makefile Signed-off-by: Yannik Fuhrmeister --- operator/Makefile | 46 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/operator/Makefile b/operator/Makefile index 8c94ed3085..fc9274a1e6 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -2,8 +2,17 @@ # # SPDX-License-Identifier: Apache-2.0 -# Image URL to use all building/pushing image targets -IMG ?= securecodebox/operator:latest +IMG_NS ?= securecodebox + +# Image URL to use all building/pushing image targets for the operator +OPERATOR_IMG ?= operator + +# Image URL to use all building/pushing image targets for the lurker +LURKER_IMG ?= lurker + +# Tag used for the images +IMG_TAG ?= sha-$$(git rev-parse --short HEAD) + # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false" @@ -67,13 +76,40 @@ run: manifests generate fmt vet ## Run a controller from your host. go run ./main.go docker-build: test ## Build docker image with the manager. - docker build -t ${IMG} . + @echo ".: ⚙️ Build Container Images" + docker build -t $(IMG_NS)/${OPERATOR_IMG}:${IMG_TAG} . + cd ../lurker && docker build -t $(IMG_NS)/$(LURKER_IMG):$(IMG_TAG) . docker-push: ## Push docker image with the manager. - docker push ${IMG} + docker push $(IMG_NS)/${OPERATOR_IMG}:${IMG_TAG} + docker push $(IMG_NS)/$(LURKER_IMG):$(IMG_TAG) + +docker-export: + @echo ".: 💾 Export Container Images" + docker save $(IMG_NS)/$(OPERATOR_IMG):$(IMG_TAG) > $(OPERATOR_IMG).tar + docker save $(IMG_NS)/$(LURKER_IMG):$(IMG_TAG) > $(LURKER_IMG).tar ##@ Deployment +kind-import: + @echo ".: 💾 Importing the image archive to local kind cluster." + kind load image-archive ./$(OPERATOR_IMG).tar + kind load image-archive ./$(LURKER_IMG).tar + +helm-deploy: + @echo ".: ⚙️ Deploying Operator with the Image tag '$(IMG_TAG)' into kind." + # If not exists create namespace where the tests will be executed + kubectl create namespace integration-tests --dry-run=client -o yaml | kubectl apply -f - + # If not exists create secureCodeBox operator namespace + kubectl create namespace securecodebox-system --dry-run=client -o yaml | kubectl apply -f - + helm -n securecodebox-system upgrade --install securecodebox-operator ./ --wait \ + --set="image.repository=docker.io/$(IMG_NS)/$(OPERATOR_IMG)" \ + --set="image.tag=$(IMG_TAG)" \ + --set="image.pullPolicy=IfNotPresent" \ + --set="lurker.image.repository=docker.io/$(IMG_NS)/$(LURKER_IMG)" \ + --set="lurker.image.tag=$(IMG_TAG)" \ + --set="lurker.pullPolicy=IfNotPresent" + install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/crd | kubectl apply -f - @@ -81,7 +117,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified $(KUSTOMIZE) build config/crd | kubectl delete -f - deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/manager && $(KUSTOMIZE) edit set image controller=${OPERATOR_IMG} $(KUSTOMIZE) build config/default | kubectl apply -f - undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. From 2f45c45d833489322063ebe54e80a4a8cdd4370d Mon Sep 17 00:00:00 2001 From: Yannik Fuhrmeister Date: Tue, 20 Jul 2021 08:32:06 +0200 Subject: [PATCH 5/9] Change Amass Build To Use Makefile Signed-off-by: Yannik Fuhrmeister --- .github/workflows/ci.yaml | 107 +++++++++++++++++++++++++++++++++----- 1 file changed, 94 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fef2f458a7..9de7de3431 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,6 +10,8 @@ env: GO_VERSION: "1.15" PYTHON_VERSION: "3.9" + NODE_VERSION: "14" + NPM_VERSION: "7" # ---- Docker Namespace ---- @@ -117,7 +119,7 @@ jobs: env: CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} with: - coverageCommand: npm test -- --ci --colors --coverage + coverageCommand: npm test -- --ci --colors --coverage --testPathIgnorePatterns=/integration-tests/ # ---- Build Stage ---- @@ -280,6 +282,97 @@ jobs: tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} +# ---- New Makefile based CI Pipeline steps ---- + + makefile-scanner: + name: "Make Scanners" + needs: + - sdk + - operator + runs-on: ubuntu-latest + strategy: + matrix: + unit: ["amass"] + steps: + - name: Checkout + uses: actions/checkout@v2 + + - uses: actions/setup-node@v2 + name: Setup Node + with: + node-version: ${{ env. NODE_VERSION }} + + - name: Update NPM + run: npm i -g npm@${{ env.NPM_VERSION }} + + - name: Install Dependencies + working-directory: ./scanners/${{ matrix.unit }}/ + run: make install-deps + + - name: Unit Tests + working-directory: ./scanners/${{ matrix.unit }}/ + run: make unit-tests + + - name: Set baseImageTag To commit Hash + run: | + echo "baseImageTag=sha-$(git rev-parse --short HEAD)" >> $GITHUB_ENV + + - name: Docker Meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + with: + images: ${{ env.DOCKER_NAMESPACE }}/parser-${{ matrix.unit }} + tag-sha: true + tag-semver: | + {{ version }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build + uses: docker/build-push-action@v2 + with: + context: ./scanners/${{ matrix.unit }}/parser/ + file: ./scanners/${{ matrix.unit }}/parser/Dockerfile + load: true + build-args: | + namespace=${{ env.DOCKER_NAMESPACE }} + baseImageTag=${{ env.baseImageTag }} + platforms: linux/amd64 + push: false + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + + - name: Export Docker Images + working-directory: ./scanners/${{ matrix.unit }}/ + run: make docker-export + + - name: "Start kind cluster" + run: | + kind version + kind create cluster --image kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6 --wait 3m + + - name: "Inspect kind cluster" + run: | + kubectl config current-context + kubectl get node + + - name: Kind Import Images + working-directory: ./scanners/${{ matrix.unit }}/ + run: make kind-import + + - name: Install Operator + working-directory: ./operator + run: | + make helm-deploy + + # ---- Build Stage | Matrix Hooks ---- hooks: @@ -353,7 +446,6 @@ jobs: strategy: matrix: parser: - - amass - angularjs-csti-scanner - git-repo-scanner - gitleaks @@ -770,17 +862,6 @@ jobs: kubectl create deployment --image nginx:alpine nginx --namespace demo-targets kubectl expose deployment nginx --port 80 --namespace demo-targets - # ---- OWASP Amass Integration Tests ---- - - - name: "amass Integration Tests" - run: | - kubectl -n integration-tests delete scans --all - helm -n integration-tests install amass ./scanners/amass/ \ - --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-amass" \ - --set="parser.image.tag=sha-$(git rev-parse --short HEAD)" - cd tests/integration/ - npx jest --ci --color scanner/amass.test.js - # ---- gitleaks Integration Tests ---- - name: "gitleaks Integration Tests" From 6bfdd8448a27bcb60d931149f3d80bc03c75ca35 Mon Sep 17 00:00:00 2001 From: Yannik Fuhrmeister Date: Tue, 20 Jul 2021 14:25:37 +0200 Subject: [PATCH 6/9] Push Image If Workflow Was Triggered By Release Signed-off-by: Yannik Fuhrmeister --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9de7de3431..dba301a007 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -345,7 +345,7 @@ jobs: namespace=${{ env.DOCKER_NAMESPACE }} baseImageTag=${{ env.baseImageTag }} platforms: linux/amd64 - push: false + push: ${{ github.event_name == 'release' }} tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} From 1f91be35730d8aee1ac4190d9913d46d2c6fb1f8 Mon Sep 17 00:00:00 2001 From: Yannik Fuhrmeister Date: Tue, 20 Jul 2021 14:50:16 +0200 Subject: [PATCH 7/9] Execute Integration Tests Signed-off-by: Yannik Fuhrmeister --- .github/workflows/ci.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dba301a007..fd2611da83 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -372,6 +372,17 @@ jobs: run: | make helm-deploy + - name: Deploy ${{ matrix.unit }} + working-directory: ./scanners/${{ matrix.unit }}/ + run: make deploy + + - name: Deploy Test Dependencies + working-directory: ./scanners/${{ matrix.unit }}/ + run: make deploy-test-deps + + - name: Start Integration Tests + working-directory: ./scanners/${{ matrix.unit }}/ + run: make integration-tests # ---- Build Stage | Matrix Hooks ---- From 241ecd786ed620bd3e5b9efe288ced7b24ab5c66 Mon Sep 17 00:00:00 2001 From: Yannik Fuhrmeister Date: Tue, 20 Jul 2021 14:50:50 +0200 Subject: [PATCH 8/9] Remove unused targets Signed-off-by: Yannik Fuhrmeister --- scanners/amass/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scanners/amass/Makefile b/scanners/amass/Makefile index bfcf7e82c0..1cae5b0810 100644 --- a/scanners/amass/Makefile +++ b/scanners/amass/Makefile @@ -57,11 +57,11 @@ parser-prefix = parser build: | install-deps docker-build -test: | unit-tests docker-export kind-import deploy-prerequisites deploy deploy-test-deps integration-tests +test: | unit-tests docker-export kind-import deploy deploy-test-deps integration-tests -all: | clean install-deps unit-tests docker-build docker-export kind-import deploy-prerequisites deploy deploy-test-deps integration-tests +all: | clean install-deps unit-tests docker-build docker-export kind-import deploy deploy-test-deps integration-tests -.PHONY: unit-tests install-deps docker-build docker-export kind-import deploy-prerequisites deploy deploy-test-deps integration-tests all build test +.PHONY: unit-tests install-deps docker-build docker-export kind-import deploy deploy-test-deps integration-tests all build test unit-tests: @echo ".: 🧪 Starting unit-tests for '$(scanner)' parser with 'jest@$(JEST_VERSION)'." From 8cdee9216e056f5a8bbac94aca45ea4e12152177 Mon Sep 17 00:00:00 2001 From: Yannik Fuhrmeister Date: Mon, 26 Jul 2021 08:48:49 +0200 Subject: [PATCH 9/9] Install Dependencies For parser-sdk Signed-off-by: Yannik Fuhrmeister --- scanners/amass/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/scanners/amass/Makefile b/scanners/amass/Makefile index 1cae5b0810..29ef87974c 100644 --- a/scanners/amass/Makefile +++ b/scanners/amass/Makefile @@ -70,6 +70,7 @@ unit-tests: install-deps: @echo ".: ⚙️ Installing all scanner specific dependencies." cd ./.. && npm ci + cd ../../parser-sdk/nodejs && npm ci cd ./parser/ && npm ci docker-build: