diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fef2f458a7..fd2611da83 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,6 +10,8 @@ env: GO_VERSION: "1.15" PYTHON_VERSION: "3.9" + NODE_VERSION: "14" + NPM_VERSION: "7" # ---- Docker Namespace ---- @@ -117,7 +119,7 @@ jobs: env: CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} with: - coverageCommand: npm test -- --ci --colors --coverage + coverageCommand: npm test -- --ci --colors --coverage --testPathIgnorePatterns=/integration-tests/ # ---- Build Stage ---- @@ -280,6 +282,108 @@ jobs: tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} +# ---- New Makefile based CI Pipeline steps ---- + + makefile-scanner: + name: "Make Scanners" + needs: + - sdk + - operator + runs-on: ubuntu-latest + strategy: + matrix: + unit: ["amass"] + steps: + - name: Checkout + uses: actions/checkout@v2 + + - uses: actions/setup-node@v2 + name: Setup Node + with: + node-version: ${{ env. NODE_VERSION }} + + - name: Update NPM + run: npm i -g npm@${{ env.NPM_VERSION }} + + - name: Install Dependencies + working-directory: ./scanners/${{ matrix.unit }}/ + run: make install-deps + + - name: Unit Tests + working-directory: ./scanners/${{ matrix.unit }}/ + run: make unit-tests + + - name: Set baseImageTag To commit Hash + run: | + echo "baseImageTag=sha-$(git rev-parse --short HEAD)" >> $GITHUB_ENV + + - name: Docker Meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + with: + images: ${{ env.DOCKER_NAMESPACE }}/parser-${{ matrix.unit }} + tag-sha: true + tag-semver: | + {{ version }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build + uses: docker/build-push-action@v2 + with: + context: ./scanners/${{ matrix.unit }}/parser/ + file: ./scanners/${{ matrix.unit }}/parser/Dockerfile + load: true + build-args: | + namespace=${{ env.DOCKER_NAMESPACE }} + baseImageTag=${{ env.baseImageTag }} + platforms: linux/amd64 + push: ${{ github.event_name == 'release' }} + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + + - name: Export Docker Images + working-directory: ./scanners/${{ matrix.unit }}/ + run: make docker-export + + - name: "Start kind cluster" + run: | + kind version + kind create cluster --image kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6 --wait 3m + + - name: "Inspect kind cluster" + run: | + kubectl config current-context + kubectl get node + + - name: Kind Import Images + working-directory: ./scanners/${{ matrix.unit }}/ + run: make kind-import + + - name: Install Operator + working-directory: ./operator + run: | + make helm-deploy + + - name: Deploy ${{ matrix.unit }} + working-directory: ./scanners/${{ matrix.unit }}/ + run: make deploy + + - name: Deploy Test Dependencies + working-directory: ./scanners/${{ matrix.unit }}/ + run: make deploy-test-deps + + - name: Start Integration Tests + working-directory: ./scanners/${{ matrix.unit }}/ + run: make integration-tests + # ---- Build Stage | Matrix Hooks ---- hooks: @@ -353,7 +457,6 @@ jobs: strategy: matrix: parser: - - amass - angularjs-csti-scanner - git-repo-scanner - gitleaks @@ -770,17 +873,6 @@ jobs: kubectl create deployment --image nginx:alpine nginx --namespace demo-targets kubectl expose deployment nginx --port 80 --namespace demo-targets - # ---- OWASP Amass Integration Tests ---- - - - name: "amass Integration Tests" - run: | - kubectl -n integration-tests delete scans --all - helm -n integration-tests install amass ./scanners/amass/ \ - --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-amass" \ - --set="parser.image.tag=sha-$(git rev-parse --short HEAD)" - cd tests/integration/ - npx jest --ci --color scanner/amass.test.js - # ---- gitleaks Integration Tests ---- - name: "gitleaks Integration Tests" diff --git a/operator/Makefile b/operator/Makefile index 8c94ed3085..fc9274a1e6 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -2,8 +2,17 @@ # # SPDX-License-Identifier: Apache-2.0 -# Image URL to use all building/pushing image targets -IMG ?= securecodebox/operator:latest +IMG_NS ?= securecodebox + +# Image URL to use all building/pushing image targets for the operator +OPERATOR_IMG ?= operator + +# Image URL to use all building/pushing image targets for the lurker +LURKER_IMG ?= lurker + +# Tag used for the images +IMG_TAG ?= sha-$$(git rev-parse --short HEAD) + # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false" @@ -67,13 +76,40 @@ run: manifests generate fmt vet ## Run a controller from your host. go run ./main.go docker-build: test ## Build docker image with the manager. - docker build -t ${IMG} . + @echo ".: โš™๏ธ Build Container Images" + docker build -t $(IMG_NS)/${OPERATOR_IMG}:${IMG_TAG} . + cd ../lurker && docker build -t $(IMG_NS)/$(LURKER_IMG):$(IMG_TAG) . docker-push: ## Push docker image with the manager. - docker push ${IMG} + docker push $(IMG_NS)/${OPERATOR_IMG}:${IMG_TAG} + docker push $(IMG_NS)/$(LURKER_IMG):$(IMG_TAG) + +docker-export: + @echo ".: ๐Ÿ’พ Export Container Images" + docker save $(IMG_NS)/$(OPERATOR_IMG):$(IMG_TAG) > $(OPERATOR_IMG).tar + docker save $(IMG_NS)/$(LURKER_IMG):$(IMG_TAG) > $(LURKER_IMG).tar ##@ Deployment +kind-import: + @echo ".: ๐Ÿ’พ Importing the image archive to local kind cluster." + kind load image-archive ./$(OPERATOR_IMG).tar + kind load image-archive ./$(LURKER_IMG).tar + +helm-deploy: + @echo ".: โš™๏ธ Deploying Operator with the Image tag '$(IMG_TAG)' into kind." + # If not exists create namespace where the tests will be executed + kubectl create namespace integration-tests --dry-run=client -o yaml | kubectl apply -f - + # If not exists create secureCodeBox operator namespace + kubectl create namespace securecodebox-system --dry-run=client -o yaml | kubectl apply -f - + helm -n securecodebox-system upgrade --install securecodebox-operator ./ --wait \ + --set="image.repository=docker.io/$(IMG_NS)/$(OPERATOR_IMG)" \ + --set="image.tag=$(IMG_TAG)" \ + --set="image.pullPolicy=IfNotPresent" \ + --set="lurker.image.repository=docker.io/$(IMG_NS)/$(LURKER_IMG)" \ + --set="lurker.image.tag=$(IMG_TAG)" \ + --set="lurker.pullPolicy=IfNotPresent" + install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/crd | kubectl apply -f - @@ -81,7 +117,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified $(KUSTOMIZE) build config/crd | kubectl delete -f - deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/manager && $(KUSTOMIZE) edit set image controller=${OPERATOR_IMG} $(KUSTOMIZE) build config/default | kubectl apply -f - undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. diff --git a/scanners/amass/.gitignore b/scanners/amass/.gitignore new file mode 100644 index 0000000000..d874ad67cc --- /dev/null +++ b/scanners/amass/.gitignore @@ -0,0 +1 @@ +*.tar diff --git a/scanners/amass/.helmignore b/scanners/amass/.helmignore index 5df91458a9..83c09e25cb 100644 --- a/scanners/amass/.helmignore +++ b/scanners/amass/.helmignore @@ -1,9 +1,36 @@ # SPDX-FileCopyrightText: 2020 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* diff --git a/scanners/amass/Makefile b/scanners/amass/Makefile new file mode 100644 index 0000000000..29ef87974c --- /dev/null +++ b/scanners/amass/Makefile @@ -0,0 +1,112 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# +# +# This Makefile is intended to be used for developement and testing only. +# For using this scanner/hook in production please use the helm chart. +# See: +# +# This Makefile expects some additional software to be installed: +# - git +# - node + npm +# - docker +# - kind +# - kubectl +# - helm + +# Thx to https://stackoverflow.com/questions/5618615/check-if-a-program-exists-from-a-makefile +EXECUTABLES = make docker kind git node npm npx kubectl helm +K := $(foreach exec,$(EXECUTABLES),\ + $(if $(shell which $(exec)),some string,$(error "ERROR: The prerequisites are not met to execute this makefile! No '$(exec)' found in your PATH"))) + + +# Variables you might want to override: +# +# IMG_NS: Defines the namespace under which the images are build. +# For `securecodebox/scanner-nmap` `securecodebox` is the namespace +# Defaults to `securecodebox` +# +# BASE_IMG_TAG: Defines the tag of the base image used to build this scanner/hook +# +# IMG_TAG: Tag used to tag the newly created image. Defaults to the shortend commit hash +# prefixed with `sha-` e.g. `sha-ef8de4b7` +# +# JEST_VERSION Defines the jest version used for executing the tests. Defaults to latest +# +# Examples: +# make all IMG_TAG=main +# make deploy IMG_TAG=$(git rev-parse --short HEAD) +# make integration-tests +# + +SHELL = /bin/sh + +IMG_NS ?= securecodebox +GIT_TAG ?= $$(git rev-parse --short HEAD) +BASE_IMG_TAG ?= latest +IMG_TAG ?= "sha-$(GIT_TAG)" +JEST_VERSION ?= latest + +scanner = amass +scanner-prefix = scanner +parser-prefix = parser + + +build: | install-deps docker-build + +test: | unit-tests docker-export kind-import deploy deploy-test-deps integration-tests + +all: | clean install-deps unit-tests docker-build docker-export kind-import deploy deploy-test-deps integration-tests + +.PHONY: unit-tests install-deps docker-build docker-export kind-import deploy deploy-test-deps integration-tests all build test + +unit-tests: + @echo ".: ๐Ÿงช Starting unit-tests for '$(scanner)' parser with 'jest@$(JEST_VERSION)'." + cd parser && npx --yes --package jest@$(JEST_VERSION) jest --ci --colors --coverage . + +install-deps: + @echo ".: โš™๏ธ Installing all scanner specific dependencies." + cd ./.. && npm ci + cd ../../parser-sdk/nodejs && npm ci + cd ./parser/ && npm ci + +docker-build: + @echo ".: โš™๏ธ Build With BASE_IMG_TAG: '$(BASE_IMG_TAG)'." + docker build --build-arg=baseImageTag=$(BASE_IMG_TAG) --build-arg=namespace=$(IMG_NS) -t $(IMG_NS)/$(parser-prefix)-$(scanner):$(IMG_TAG) -f ./parser/Dockerfile ./parser + +docker-export: + @echo ".: โš™๏ธ Saving new docker image archive to '$(parser-prefix)-$(scanner).tar'." + docker save $(IMG_NS)/$(parser-prefix)-$(scanner):$(IMG_TAG) -o $(parser-prefix)-$(scanner).tar + +kind-import: + @echo ".: ๐Ÿ’พ Importing the image archive '$(parser-prefix)-$(scanner).tar' to local kind cluster." + kind load image-archive ./$(parser-prefix)-$(scanner).tar + +deploy: + @echo ".: ๐Ÿ’พ Deploying '$(scanner)' scanner HelmChart with the docker tag '$(IMG_TAG)' into kind namespace 'integration-tests'." + helm -n integration-tests upgrade --install $(scanner) ./ --wait \ + --set="parser.image.repository=docker.io/$(IMG_NS)/$(parser-prefix)-$(scanner)" \ + --set="parser.image.tag=$(IMG_TAG)" + +deploy-test-deps: + +install-integration-test-deps: + +integration-tests: + @echo ".: ๐Ÿฉบ Starting integration test in kind namespace 'integration-tests'." + kubectl -n integration-tests delete scans --all + cd ../../tests/integration/ && npm ci + npx --yes --package jest@$(JEST_VERSION) jest --ci --colors --coverage ./integration-tests + +clean: + @echo ".: ๐Ÿงน Cleaning up all generated files." + rm -f ./$(parser-prefix)-$(scanner).tar + rm -rf ./parser/node_modules + rm -rf ./parser/coverage + rm -rf ./integration-tests/node_modules + rm -rf ./integration-tests/coverage + rm -rf ../node_modules + rm -rf ../coverage diff --git a/tests/integration/scanner/amass.test.js b/scanners/amass/integration-tests/amass.test.js similarity index 86% rename from tests/integration/scanner/amass.test.js rename to scanners/amass/integration-tests/amass.test.js index 7f1c5b5914..d23fc2d343 100644 --- a/tests/integration/scanner/amass.test.js +++ b/scanners/amass/integration-tests/amass.test.js @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -const { scan } = require("../helpers"); +const { scan } = require("../../../tests/integration/helpers.js"); jest.retryTimes(3); diff --git a/scanners/amass/integration-tests/jest.config.json b/scanners/amass/integration-tests/jest.config.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/scanners/amass/integration-tests/jest.config.json @@ -0,0 +1 @@ +{} diff --git a/scanners/amass/parser/jest.config.json b/scanners/amass/parser/jest.config.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/scanners/amass/parser/jest.config.json @@ -0,0 +1 @@ +{}