From 42e6910b1c3800ed67c1f45a033e25b34395cdd1 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Wed, 28 Dec 2022 17:47:23 +0530 Subject: [PATCH 01/13] CLI for SS --- Dockerfile-cli | 12 ++++++++++++ Makefile | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 Dockerfile-cli diff --git a/Dockerfile-cli b/Dockerfile-cli new file mode 100644 index 0000000..4eb62e5 --- /dev/null +++ b/Dockerfile-cli @@ -0,0 +1,12 @@ +FROM golang:1.19.4-bullseye +RUN apt update +RUN apt install -y libboost-dev cmake git ragel protoc-gen-go +WORKDIR /tmp/src +RUN git clone https://github.com/deepfence/SecretScanner.git +RUN git clone https://github.com/intel/hyperscan.git +RUN cd hyperscan && mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=MinSizeRel +WORKDIR /tmp/src/SecretScanner +RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1 \ + && go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 +RUN make clean +RUN make diff --git a/Makefile b/Makefile index 21acc41..eaa1a98 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,6 @@ clean: -rm ./SecretScanner SecretScanner: $(PWD)/**/*.go $(PWD)/agent-plugins-grpc/proto/*.go - go build -buildvcs=false -v . + PKG_CONFIG_PATH=/tmp/src/hyperscan/build CGO_LDFLAGS="-L /tmp/src/hyperscan/build/lib -static" CGO_CFLAGS="-I/tmp/src/hyperscan/src" go build ./main.go .PHONY: clean From 2d500ce032e7bbec360ed0251eb600d5cb651c08 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Wed, 28 Dec 2022 18:00:39 +0530 Subject: [PATCH 02/13] branch name change --- Dockerfile-cli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-cli b/Dockerfile-cli index 4eb62e5..afd7acd 100644 --- a/Dockerfile-cli +++ b/Dockerfile-cli @@ -2,7 +2,7 @@ FROM golang:1.19.4-bullseye RUN apt update RUN apt install -y libboost-dev cmake git ragel protoc-gen-go WORKDIR /tmp/src -RUN git clone https://github.com/deepfence/SecretScanner.git +RUN git clone --branch cli https://github.com/deepfence/SecretScanner.git RUN git clone https://github.com/intel/hyperscan.git RUN cd hyperscan && mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=MinSizeRel WORKDIR /tmp/src/SecretScanner From 3eadc7ee2d46fb3a98104b477196a7fdb2070793 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Wed, 28 Dec 2022 18:18:35 +0530 Subject: [PATCH 03/13] removed main.go compilation --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index eaa1a98..8eeee88 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,6 @@ clean: -rm ./SecretScanner SecretScanner: $(PWD)/**/*.go $(PWD)/agent-plugins-grpc/proto/*.go - PKG_CONFIG_PATH=/tmp/src/hyperscan/build CGO_LDFLAGS="-L /tmp/src/hyperscan/build/lib -static" CGO_CFLAGS="-I/tmp/src/hyperscan/src" go build ./main.go + PKG_CONFIG_PATH=/tmp/src/hyperscan/build CGO_LDFLAGS="-L /tmp/src/hyperscan/build/lib -static" CGO_CFLAGS="-I/tmp/src/hyperscan/src" go build .PHONY: clean From b0346d1f1ee03d7ba049ebf061380ac56f561ab9 Mon Sep 17 00:00:00 2001 From: Thomas Legris Date: Wed, 28 Dec 2022 21:49:22 +0900 Subject: [PATCH 04/13] Clean up Makefile --- Dockerfile | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d949ecf..718a6f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1 \ WORKDIR /home/deepfence/src/SecretScanner COPY . . RUN make clean -RUN make +RUN PKG_CONFIG_PATH=/tmp/src/hyperscan/build CGO_LDFLAGS="-L /tmp/src/hyperscan/build/lib -static" CGO_CFLAGS="-I/tmp/src/hyperscan/src" make FROM alpine:3.15 MAINTAINER DeepFence diff --git a/Makefile b/Makefile index 8eeee88..9d59821 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,6 @@ clean: -rm ./SecretScanner SecretScanner: $(PWD)/**/*.go $(PWD)/agent-plugins-grpc/proto/*.go - PKG_CONFIG_PATH=/tmp/src/hyperscan/build CGO_LDFLAGS="-L /tmp/src/hyperscan/build/lib -static" CGO_CFLAGS="-I/tmp/src/hyperscan/src" go build + go build .PHONY: clean From 2fe947cfa1b92518d05e32f3a1904e4dc62feb9e Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Wed, 28 Dec 2022 18:36:52 +0530 Subject: [PATCH 05/13] cli bullseye to alpine --- Dockerfile-cli | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile-cli b/Dockerfile-cli index afd7acd..9ac9f44 100644 --- a/Dockerfile-cli +++ b/Dockerfile-cli @@ -1,6 +1,6 @@ -FROM golang:1.19.4-bullseye -RUN apt update -RUN apt install -y libboost-dev cmake git ragel protoc-gen-go +FROM golang:1.19.4-alpine3.17 +RUN apk update +RUN apk add -y libboost-dev cmake git ragel protoc WORKDIR /tmp/src RUN git clone --branch cli https://github.com/deepfence/SecretScanner.git RUN git clone https://github.com/intel/hyperscan.git From 3099dc0a24439e4d1009bd2a6523b4cc36d00f9c Mon Sep 17 00:00:00 2001 From: Thomas Legris Date: Wed, 28 Dec 2022 22:28:46 +0900 Subject: [PATCH 06/13] Add new static Makefile action --- Dockerfile | 2 +- Dockerfile-static | 24 ++++++++++++++++++++++++ Makefile | 8 ++++++-- agent-plugins-grpc | 2 +- 4 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 Dockerfile-static diff --git a/Dockerfile b/Dockerfile index 718a6f9..d949ecf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1 \ WORKDIR /home/deepfence/src/SecretScanner COPY . . RUN make clean -RUN PKG_CONFIG_PATH=/tmp/src/hyperscan/build CGO_LDFLAGS="-L /tmp/src/hyperscan/build/lib -static" CGO_CFLAGS="-I/tmp/src/hyperscan/src" make +RUN make FROM alpine:3.15 MAINTAINER DeepFence diff --git a/Dockerfile-static b/Dockerfile-static new file mode 100644 index 0000000..5047cf6 --- /dev/null +++ b/Dockerfile-static @@ -0,0 +1,24 @@ +FROM golang:1.19.4-alpine + +RUN apk add --upgrade gcc musl-dev pkgconfig g++ make git protoc cmake boost-dev +RUN apk add hyperscan-dev --repository=https://dl-cdn.alpinelinux.org/alpine/v3.13/community +RUN apk add ragel + +ENV PKG_CONFIG_PATH=/usr/local/include/hs/ \ + CGO_CFLAGS="-I/usr/local/include/hyperscan/src" \ + LD_LIBRARY_PATH=/usr/local/lib:/usr/local/include/hs/lib:$LD_LIBRARY_PATH + +RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1 \ + && go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 + +WORKDIR /go/src +RUN git clone https://github.com/intel/hyperscan.git +RUN cd hyperscan && mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=MinSizeRel + +WORKDIR /go/src/SecretScanner +ENV PKG_CONFIG_PATH=/go/src/hyperscan/build +ENV CGO_LDFLAGS="-L /go/src/hyperscan/build/lib -static" +ENV CGO_CFLAGS="-I/go/src/hyperscan/src" +ENV GO_BUILD_EXTRA="-buildvcs=false" + +ENTRYPOINT ["/bin/sh", "-c", "make clean && make"] diff --git a/Makefile b/Makefile index 9d59821..62b0df9 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,10 @@ clean: -rm ./SecretScanner SecretScanner: $(PWD)/**/*.go $(PWD)/agent-plugins-grpc/proto/*.go - go build + go build $(GO_BUILD_EXTRA) -.PHONY: clean +static: + docker build -t static-secret-scanner -f Dockerfile-static . + docker run -v $(PWD):/go/src/SecretScanner static-secret-scanner + +.PHONY: clean static diff --git a/agent-plugins-grpc b/agent-plugins-grpc index 6584ed0..6475832 160000 --- a/agent-plugins-grpc +++ b/agent-plugins-grpc @@ -1 +1 @@ -Subproject commit 6584ed03e8ccf98af554015a2363bb627a23f84c +Subproject commit 6475832e96845428231a3b047b80ac79678d76a7 From 82143d3756f542771bd23518a1979b69d02f2762 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Thu, 29 Dec 2022 13:16:43 +0530 Subject: [PATCH 07/13] Adding debug logs, to be reverted --- scan/process_image.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scan/process_image.go b/scan/process_image.go index a26b9a7..f4d357a 100644 --- a/scan/process_image.go +++ b/scan/process_image.go @@ -94,6 +94,7 @@ func (imageScan *ImageScan) scan() ([]output.SecretFound, error) { defer core.DeleteTmpDir(tempDir) tempSecretsFound, err := imageScan.processImageLayers(tempDir) + fmt.Printf("length of tempSecretsFound at 97 processImage: %d \n", len(tempSecretsFound)) if err != nil { core.GetSession().Log.Error("scanImage: %s", err) return tempSecretsFound, err @@ -136,7 +137,7 @@ func ScanSecretsInDir(layer string, baseDir string, fullDir string, isFirstSecre var scanDirPath string if layer != "" { - scanDirPath = strings.TrimPrefix(path, baseDir + "/" + layer) + scanDirPath = strings.TrimPrefix(path, baseDir+"/"+layer) if scanDirPath == "" { scanDirPath = "/" } @@ -184,6 +185,7 @@ func ScanSecretsInDir(layer string, baseDir string, fullDir string, isFirstSecre // fmt.Println(relPath, file.Filename, file.Extension, layer) secrets, err = signature.MatchPatternSignatures(contents, relPath, file.Filename, file.Extension, layer, numSecrets, matchedRuleSet) + fmt.Printf("length of secrets at 188 processImage: %d \n", len(secrets)) if err != nil { session.Log.Info("relPath: %s, Filename: %s, Extension: %s, layer: %s", relPath, file.Filename, file.Extension, layer) @@ -194,14 +196,16 @@ func ScanSecretsInDir(layer string, baseDir string, fullDir string, isFirstSecre output.PrintColoredSecrets(secrets, isFirstSecret) } tempSecretsFound = append(tempSecretsFound, secrets...) + fmt.Printf("length of tempSecretsFound at 199 processImage: %d \n", len(tempSecretsFound)) } secrets = signature.MatchSimpleSignatures(relPath, file.Filename, file.Extension, layer, numSecrets) + fmt.Printf("length of secrets at 203 processImage: %d \n", len(secrets)) if *session.Options.Quiet { output.PrintColoredSecrets(secrets, isFirstSecret) } tempSecretsFound = append(tempSecretsFound, secrets...) - + fmt.Printf("length of tempSecretsFound at 208 processImage: %d \n", len(tempSecretsFound)) // Don't report secrets if number of secrets exceeds MAX value if *numSecrets >= *session.Options.MaxSecrets { return maxSecretsExceeded @@ -217,6 +221,7 @@ func ScanSecretsInDir(layer string, baseDir string, fullDir string, isFirstSecre fmt.Printf("Error in filepath.Walk: %s\n", walkErr) } } + fmt.Printf("length of tempSecretsFound at 224 processImage: %d \n", len(tempSecretsFound)) return tempSecretsFound, nil } @@ -263,7 +268,9 @@ func (imageScan *ImageScan) processImageLayers(imageManifestPath string) ([]outp } core.GetSession().Log.Debug("Analyzing dir: %s", targetDir) secrets, err = ScanSecretsInDir(layerIDs[i], extractPath, targetDir, &isFirstSecret, &imageScan.numSecrets, matchedRuleSet) + fmt.Printf("length of secrets at 267 processImage: %d \n", len(secrets)) tempSecretsFound = append(tempSecretsFound, secrets...) + fmt.Printf("length of tempSecretsFound at 269 processImage: %d \n", len(tempSecretsFound)) if err != nil { core.GetSession().Log.Error("ProcessImageLayers: %s", err) // return tempSecretsFound, err @@ -378,7 +385,7 @@ func untar(tarName string, xpath string) (err error) { relPath := strings.Split(fileName, "/") var absDirPath string if len(relPath) > 1 { - dirs := relPath[0: len(relPath) - 1] + dirs := relPath[0 : len(relPath)-1] absDirPath = filepath.Join(absPath, strings.Join(dirs, "/")) } if err := os.MkdirAll(absDirPath, 0755); err != nil { @@ -402,7 +409,7 @@ func untar(tarName string, xpath string) (err error) { // fmt.Printf("x %s\n", absFileName) n, cpErr := io.Copy(file, tr) if closeErr := file.Close(); closeErr != nil { // close file immediately - fmt.Println("clserr:"+closeErr.Error()) + fmt.Println("clserr:" + closeErr.Error()) return err } if cpErr != nil { From af59d96005778eed1bb0359599efdcce72540882 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Thu, 29 Dec 2022 13:33:42 +0530 Subject: [PATCH 08/13] adding log for revert --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 48a9753..2c1bac8 100644 --- a/main.go +++ b/main.go @@ -81,6 +81,7 @@ func findSecretsInDir(dir string) (*output.JsonDirSecretsOutput, error) { var numSecrets uint = 0 secrets, err := scan.ScanSecretsInDir("", "", dir, &isFirstSecret, &numSecrets, nil) + fmt.Printf("length of secrets at 84 main: %d \n", len(secrets)) if err != nil { core.GetSession().Log.Error("findSecretsInDir: %s", err) return nil, err From 22b0395c2e8704f74cb36fd2c3106ef4c2276278 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Thu, 29 Dec 2022 14:12:57 +0530 Subject: [PATCH 09/13] reverting debug logs and cleaning up --- Dockerfile-cli | 12 ------------ main.go | 1 - scan/process_image.go | 15 ++++----------- 3 files changed, 4 insertions(+), 24 deletions(-) delete mode 100644 Dockerfile-cli diff --git a/Dockerfile-cli b/Dockerfile-cli deleted file mode 100644 index 9ac9f44..0000000 --- a/Dockerfile-cli +++ /dev/null @@ -1,12 +0,0 @@ -FROM golang:1.19.4-alpine3.17 -RUN apk update -RUN apk add -y libboost-dev cmake git ragel protoc -WORKDIR /tmp/src -RUN git clone --branch cli https://github.com/deepfence/SecretScanner.git -RUN git clone https://github.com/intel/hyperscan.git -RUN cd hyperscan && mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=MinSizeRel -WORKDIR /tmp/src/SecretScanner -RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1 \ - && go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 -RUN make clean -RUN make diff --git a/main.go b/main.go index 2c1bac8..48a9753 100644 --- a/main.go +++ b/main.go @@ -81,7 +81,6 @@ func findSecretsInDir(dir string) (*output.JsonDirSecretsOutput, error) { var numSecrets uint = 0 secrets, err := scan.ScanSecretsInDir("", "", dir, &isFirstSecret, &numSecrets, nil) - fmt.Printf("length of secrets at 84 main: %d \n", len(secrets)) if err != nil { core.GetSession().Log.Error("findSecretsInDir: %s", err) return nil, err diff --git a/scan/process_image.go b/scan/process_image.go index f4d357a..a26b9a7 100644 --- a/scan/process_image.go +++ b/scan/process_image.go @@ -94,7 +94,6 @@ func (imageScan *ImageScan) scan() ([]output.SecretFound, error) { defer core.DeleteTmpDir(tempDir) tempSecretsFound, err := imageScan.processImageLayers(tempDir) - fmt.Printf("length of tempSecretsFound at 97 processImage: %d \n", len(tempSecretsFound)) if err != nil { core.GetSession().Log.Error("scanImage: %s", err) return tempSecretsFound, err @@ -137,7 +136,7 @@ func ScanSecretsInDir(layer string, baseDir string, fullDir string, isFirstSecre var scanDirPath string if layer != "" { - scanDirPath = strings.TrimPrefix(path, baseDir+"/"+layer) + scanDirPath = strings.TrimPrefix(path, baseDir + "/" + layer) if scanDirPath == "" { scanDirPath = "/" } @@ -185,7 +184,6 @@ func ScanSecretsInDir(layer string, baseDir string, fullDir string, isFirstSecre // fmt.Println(relPath, file.Filename, file.Extension, layer) secrets, err = signature.MatchPatternSignatures(contents, relPath, file.Filename, file.Extension, layer, numSecrets, matchedRuleSet) - fmt.Printf("length of secrets at 188 processImage: %d \n", len(secrets)) if err != nil { session.Log.Info("relPath: %s, Filename: %s, Extension: %s, layer: %s", relPath, file.Filename, file.Extension, layer) @@ -196,16 +194,14 @@ func ScanSecretsInDir(layer string, baseDir string, fullDir string, isFirstSecre output.PrintColoredSecrets(secrets, isFirstSecret) } tempSecretsFound = append(tempSecretsFound, secrets...) - fmt.Printf("length of tempSecretsFound at 199 processImage: %d \n", len(tempSecretsFound)) } secrets = signature.MatchSimpleSignatures(relPath, file.Filename, file.Extension, layer, numSecrets) - fmt.Printf("length of secrets at 203 processImage: %d \n", len(secrets)) if *session.Options.Quiet { output.PrintColoredSecrets(secrets, isFirstSecret) } tempSecretsFound = append(tempSecretsFound, secrets...) - fmt.Printf("length of tempSecretsFound at 208 processImage: %d \n", len(tempSecretsFound)) + // Don't report secrets if number of secrets exceeds MAX value if *numSecrets >= *session.Options.MaxSecrets { return maxSecretsExceeded @@ -221,7 +217,6 @@ func ScanSecretsInDir(layer string, baseDir string, fullDir string, isFirstSecre fmt.Printf("Error in filepath.Walk: %s\n", walkErr) } } - fmt.Printf("length of tempSecretsFound at 224 processImage: %d \n", len(tempSecretsFound)) return tempSecretsFound, nil } @@ -268,9 +263,7 @@ func (imageScan *ImageScan) processImageLayers(imageManifestPath string) ([]outp } core.GetSession().Log.Debug("Analyzing dir: %s", targetDir) secrets, err = ScanSecretsInDir(layerIDs[i], extractPath, targetDir, &isFirstSecret, &imageScan.numSecrets, matchedRuleSet) - fmt.Printf("length of secrets at 267 processImage: %d \n", len(secrets)) tempSecretsFound = append(tempSecretsFound, secrets...) - fmt.Printf("length of tempSecretsFound at 269 processImage: %d \n", len(tempSecretsFound)) if err != nil { core.GetSession().Log.Error("ProcessImageLayers: %s", err) // return tempSecretsFound, err @@ -385,7 +378,7 @@ func untar(tarName string, xpath string) (err error) { relPath := strings.Split(fileName, "/") var absDirPath string if len(relPath) > 1 { - dirs := relPath[0 : len(relPath)-1] + dirs := relPath[0: len(relPath) - 1] absDirPath = filepath.Join(absPath, strings.Join(dirs, "/")) } if err := os.MkdirAll(absDirPath, 0755); err != nil { @@ -409,7 +402,7 @@ func untar(tarName string, xpath string) (err error) { // fmt.Printf("x %s\n", absFileName) n, cpErr := io.Copy(file, tr) if closeErr := file.Close(); closeErr != nil { // close file immediately - fmt.Println("clserr:" + closeErr.Error()) + fmt.Println("clserr:"+closeErr.Error()) return err } if cpErr != nil { From f5b2a6d06f755cb000a1de1f9d412fcb33533f80 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Thu, 29 Dec 2022 14:14:41 +0530 Subject: [PATCH 10/13] Fixing logic of quiet --- core/options.go | 2 +- scan/process_image.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/options.go b/core/options.go index 5d86d37..f41b471 100644 --- a/core/options.go +++ b/core/options.go @@ -66,7 +66,7 @@ func ParseOptions() (*Options, error) { MaxSecrets: flag.Uint("max-secrets", 1000, "Maximum number of secrets to find in one container image or file system."), ContainerId: flag.String("container-id", "", "Id of existing container ID"), ContainerNS: flag.String("container-ns", "", "Namespace of existing container to scan, empty for docker runtime"), - Quiet: flag.Bool("quiet", false, "Don't display any output in stdout"), + Quiet: flag.Bool("quiet", true, "Don't display any output in stdout"), } flag.Var(options.ConfigPath, "config-path", "Searches for config.yaml from given directory. If not set, tries to find it from SecretScanner binary's and current directory. Can be specified multiple times.") flag.Parse() diff --git a/scan/process_image.go b/scan/process_image.go index a26b9a7..734587a 100644 --- a/scan/process_image.go +++ b/scan/process_image.go @@ -136,7 +136,7 @@ func ScanSecretsInDir(layer string, baseDir string, fullDir string, isFirstSecre var scanDirPath string if layer != "" { - scanDirPath = strings.TrimPrefix(path, baseDir + "/" + layer) + scanDirPath = strings.TrimPrefix(path, baseDir+"/"+layer) if scanDirPath == "" { scanDirPath = "/" } @@ -190,14 +190,14 @@ func ScanSecretsInDir(layer string, baseDir string, fullDir string, isFirstSecre session.Log.Error("scanSecretsInDir: %s", err) // return tempSecretsFound, err } - if *session.Options.Quiet { + if !*session.Options.Quiet { output.PrintColoredSecrets(secrets, isFirstSecret) } tempSecretsFound = append(tempSecretsFound, secrets...) } secrets = signature.MatchSimpleSignatures(relPath, file.Filename, file.Extension, layer, numSecrets) - if *session.Options.Quiet { + if !*session.Options.Quiet { output.PrintColoredSecrets(secrets, isFirstSecret) } tempSecretsFound = append(tempSecretsFound, secrets...) @@ -378,7 +378,7 @@ func untar(tarName string, xpath string) (err error) { relPath := strings.Split(fileName, "/") var absDirPath string if len(relPath) > 1 { - dirs := relPath[0: len(relPath) - 1] + dirs := relPath[0 : len(relPath)-1] absDirPath = filepath.Join(absPath, strings.Join(dirs, "/")) } if err := os.MkdirAll(absDirPath, 0755); err != nil { @@ -402,7 +402,7 @@ func untar(tarName string, xpath string) (err error) { // fmt.Printf("x %s\n", absFileName) n, cpErr := io.Copy(file, tr) if closeErr := file.Close(); closeErr != nil { // close file immediately - fmt.Println("clserr:"+closeErr.Error()) + fmt.Println("clserr:" + closeErr.Error()) return err } if cpErr != nil { From 36daa50f6912a62e0e167d7c592672c8664cc075 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Thu, 29 Dec 2022 14:50:02 +0530 Subject: [PATCH 11/13] Fixed json printing of secrets --- core/util.go | 2 +- main.go | 39 ++++++++++++++++++++++++--------------- output/output.go | 9 +++++---- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/core/util.go b/core/util.go index b0cbabe..d0f8297 100644 --- a/core/util.go +++ b/core/util.go @@ -65,7 +65,7 @@ func GetJsonFilepath(input string) (string, error) { } } if JsonFilename == "" { - JsonFilename = getSanitizedString(input) + "-secrets.json" + JsonFilename = getSanitizedString(input) + "secrets.json" } jsonFilePath := filepath.Join(outputDir, JsonFilename) GetSession().Log.Info("Complete json file path and name: %s", jsonFilePath) diff --git a/main.go b/main.go index 48a9753..34d979b 100644 --- a/main.go +++ b/main.go @@ -57,15 +57,18 @@ var session = core.GetSession() // Error, if any. Otherwise, returns nil func findSecretsInImage(image string) (*output.JsonImageSecretsOutput, error) { + jsonImageSecretsOutput := output.JsonImageSecretsOutput{ImageName: image} + jsonImageSecretsOutput.SetTime() + if !*session.Options.Quiet { + jsonImageSecretsOutput.PrintJsonHeader() + } res, err := scan.ExtractAndScanImage(image) if err != nil { return nil, err } - jsonImageSecretsOutput := output.JsonImageSecretsOutput{ImageName: image} - jsonImageSecretsOutput.SetTime() - jsonImageSecretsOutput.SetImageId(res.ImageId) - jsonImageSecretsOutput.PrintJsonHeader() - jsonImageSecretsOutput.PrintJsonFooter() + if !*session.Options.Quiet { + jsonImageSecretsOutput.PrintJsonFooter() + } jsonImageSecretsOutput.SetSecrets(res.Secrets) return &jsonImageSecretsOutput, nil @@ -79,17 +82,20 @@ func findSecretsInImage(image string) (*output.JsonImageSecretsOutput, error) { func findSecretsInDir(dir string) (*output.JsonDirSecretsOutput, error) { var isFirstSecret bool = true var numSecrets uint = 0 + jsonDirSecretsOutput := output.JsonDirSecretsOutput{DirName: *session.Options.Local} + jsonDirSecretsOutput.SetTime() + if !*session.Options.Quiet { + jsonDirSecretsOutput.PrintJsonHeader() + } secrets, err := scan.ScanSecretsInDir("", "", dir, &isFirstSecret, &numSecrets, nil) if err != nil { core.GetSession().Log.Error("findSecretsInDir: %s", err) return nil, err } - - jsonDirSecretsOutput := output.JsonDirSecretsOutput{DirName: *session.Options.Local} - jsonDirSecretsOutput.SetTime() - jsonDirSecretsOutput.PrintJsonHeader() - jsonDirSecretsOutput.PrintJsonFooter() + if !*session.Options.Quiet { + jsonDirSecretsOutput.PrintJsonFooter() + } jsonDirSecretsOutput.SetSecrets(secrets) return &jsonDirSecretsOutput, nil @@ -102,15 +108,18 @@ func findSecretsInDir(dir string) (*output.JsonDirSecretsOutput, error) { // Error, if any. Otherwise, returns nil func findSecretsInContainer(containerId string, containerNS string) (*output.JsonImageSecretsOutput, error) { + jsonImageSecretsOutput := output.JsonImageSecretsOutput{ContainerId: containerId} + jsonImageSecretsOutput.SetTime() + if !*session.Options.Quiet { + jsonImageSecretsOutput.PrintJsonHeader() + } res, err := scan.ExtractAndScanContainer(containerId, containerNS) if err != nil { return nil, err } - jsonImageSecretsOutput := output.JsonImageSecretsOutput{ContainerId: containerId} - jsonImageSecretsOutput.SetTime() - jsonImageSecretsOutput.SetImageId(res.ContainerId) - jsonImageSecretsOutput.PrintJsonHeader() - jsonImageSecretsOutput.PrintJsonFooter() + if !*session.Options.Quiet { + jsonImageSecretsOutput.PrintJsonFooter() + } jsonImageSecretsOutput.SetSecrets(res.Secrets) return &jsonImageSecretsOutput, nil diff --git a/output/output.go b/output/output.go index 4b630af..162ae89 100644 --- a/output/output.go +++ b/output/output.go @@ -43,11 +43,11 @@ type JsonDirSecretsOutput struct { } type JsonImageSecretsOutput struct { - Timestamp time.Time - ImageName string `json:"Image Name"` - ImageId string `json:"Image ID"` + Timestamp time.Time + ImageName string `json:"Image Name"` + ImageId string `json:"Image ID"` ContainerId string `json:"Container ID"` - Secrets []SecretFound + Secrets []SecretFound } func (imageOutput *JsonImageSecretsOutput) SetImageName(imageName string) { @@ -111,6 +111,7 @@ func (imageOutput JsonImageSecretsOutput) PrintJsonHeader() { fmt.Printf(Indent+"\"Timestamp\": \"%s\",\n", time.Now().Format("2006-01-02 15:04:05.000000000 -07:00")) fmt.Printf(Indent+"\"Image Name\": \"%s\",\n", imageOutput.ImageName) fmt.Printf(Indent+"\"Image ID\": \"%s\",\n", imageOutput.ImageId) + fmt.Printf(Indent+"\"Container ID\": \"%s\",\n", imageOutput.ContainerId) fmt.Printf(Indent + "\"Secrets\": [\n") } From dc6cefb7fb3048d2fa271271466502096b3a2732 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Thu, 29 Dec 2022 15:07:42 +0530 Subject: [PATCH 12/13] fix --- main.go | 12 ++++++------ scan/process_image.go | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/main.go b/main.go index 34d979b..92e1b28 100644 --- a/main.go +++ b/main.go @@ -59,14 +59,14 @@ func findSecretsInImage(image string) (*output.JsonImageSecretsOutput, error) { jsonImageSecretsOutput := output.JsonImageSecretsOutput{ImageName: image} jsonImageSecretsOutput.SetTime() - if !*session.Options.Quiet { + if !(*session.Options.Quiet) { jsonImageSecretsOutput.PrintJsonHeader() } res, err := scan.ExtractAndScanImage(image) if err != nil { return nil, err } - if !*session.Options.Quiet { + if !(*session.Options.Quiet) { jsonImageSecretsOutput.PrintJsonFooter() } jsonImageSecretsOutput.SetSecrets(res.Secrets) @@ -84,7 +84,7 @@ func findSecretsInDir(dir string) (*output.JsonDirSecretsOutput, error) { var numSecrets uint = 0 jsonDirSecretsOutput := output.JsonDirSecretsOutput{DirName: *session.Options.Local} jsonDirSecretsOutput.SetTime() - if !*session.Options.Quiet { + if !(*session.Options.Quiet) { jsonDirSecretsOutput.PrintJsonHeader() } @@ -93,7 +93,7 @@ func findSecretsInDir(dir string) (*output.JsonDirSecretsOutput, error) { core.GetSession().Log.Error("findSecretsInDir: %s", err) return nil, err } - if !*session.Options.Quiet { + if !(*session.Options.Quiet) { jsonDirSecretsOutput.PrintJsonFooter() } jsonDirSecretsOutput.SetSecrets(secrets) @@ -110,14 +110,14 @@ func findSecretsInContainer(containerId string, containerNS string) (*output.Jso jsonImageSecretsOutput := output.JsonImageSecretsOutput{ContainerId: containerId} jsonImageSecretsOutput.SetTime() - if !*session.Options.Quiet { + if !(*session.Options.Quiet) { jsonImageSecretsOutput.PrintJsonHeader() } res, err := scan.ExtractAndScanContainer(containerId, containerNS) if err != nil { return nil, err } - if !*session.Options.Quiet { + if !(*session.Options.Quiet) { jsonImageSecretsOutput.PrintJsonFooter() } jsonImageSecretsOutput.SetSecrets(res.Secrets) diff --git a/scan/process_image.go b/scan/process_image.go index 734587a..c16d5a9 100644 --- a/scan/process_image.go +++ b/scan/process_image.go @@ -190,14 +190,14 @@ func ScanSecretsInDir(layer string, baseDir string, fullDir string, isFirstSecre session.Log.Error("scanSecretsInDir: %s", err) // return tempSecretsFound, err } - if !*session.Options.Quiet { + if !(*session.Options.Quiet) { output.PrintColoredSecrets(secrets, isFirstSecret) } tempSecretsFound = append(tempSecretsFound, secrets...) } secrets = signature.MatchSimpleSignatures(relPath, file.Filename, file.Extension, layer, numSecrets) - if !*session.Options.Quiet { + if !(*session.Options.Quiet) { output.PrintColoredSecrets(secrets, isFirstSecret) } tempSecretsFound = append(tempSecretsFound, secrets...) From e807224b69ffd4c74b4ed3a371d83abf20a01576 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Sat, 31 Dec 2022 06:25:22 +0530 Subject: [PATCH 13/13] Added docs to use cli --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index bc9cd58..83d9c43 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,21 @@ docker pull node:8.11 docker run -it --rm --name=deepfence-secretscanner -v $(pwd):/home/deepfence/output -v /var/run/docker.sock:/var/run/docker.sock deepfenceio/deepfence_secret_scanner:latest -image-name node:8.11 ``` +# CLI + +Download the binary and yaml file. The binary can run to perform secret scans: +Scan a local directory +```shell +./SecretScanner -local -quiet=false -config-path +``` +Scan a local docker image +```shell +./SecretScanner -quiet=false -image-name -config-path +``` +Scan the filesystem of a running container using id or name +```shell +./SecretScanner -quiet=false -config-path -container-id +``` # Credits We have built upon the configuration file from [shhgit](https://github.com/eth0izzle/shhgit) project.