Fixes #368 - jars larger than a gig are extracted to disk when scanning (#400)

Former-commit-id: 3a0dc73
Former-commit-id: 518cc97ebcc70f4785817bcd15ef2b572a45c31b

Author: breadchris

tools/log4shell/analyze/analyze.go

@@ -15,7 +15,6 @@
 package analyze
 
 import (
-	"archive/zip"
 	"github.com/lunasec-io/lunasec/tools/log4shell/constants"
 	"github.com/lunasec-io/lunasec/tools/log4shell/types"
 	"github.com/lunasec-io/lunasec/tools/log4shell/util"
@@ -25,8 +24,11 @@ import (
 	"strings"
 )
 
-func GetJndiLookupHash(zipReader *zip.Reader, filePath string) (fileHash string) {
-	reader, err := zipReader.Open(constants.JndiLookupClasspath)
+func GetJndiLookupHash(
+	resolveArchiveFile types.ResolveArchiveFile,
+	filePath string,
+) (fileHash string) {
+	reader, err := resolveArchiveFile(constants.JndiLookupClasspath)
 	if err != nil {
 		log.Debug().
 			Str("fieName", constants.JndiLookupClasspath).
@@ -49,7 +51,11 @@ func GetJndiLookupHash(zipReader *zip.Reader, filePath string) (fileHash string)
 	return
 }
 
-func ProcessArchiveFile(zipReader *zip.Reader, reader io.Reader, filePath, fileName string) (finding *types.Finding) {
+func ProcessArchiveFile(
+	resolveArchiveFile types.ResolveArchiveFile,
+	reader io.Reader,
+	filePath, fileName string,
+) (finding *types.Finding) {
 	var (
 		jndiLookupFileHash string
 	)
@@ -93,7 +99,7 @@ func ProcessArchiveFile(zipReader *zip.Reader, reader io.Reader, filePath, fileN
 	}
 
 	if VersionIsInRange(archiveName, semverVersion, constants.JndiLookupPatchFileVersions) {
-		jndiLookupFileHash = GetJndiLookupHash(zipReader, filePath)
+		jndiLookupFileHash = GetJndiLookupHash(resolveArchiveFile, filePath)
 	}
 
 	log.Log().

tools/log4shell/commands/patch.go

@@ -38,6 +38,7 @@ func JavaArchivePatchCommand(
 
 	forcePatch := c.Bool("force-patch")
 	dryRun := c.Bool("dry-run")
+	backup := c.Bool("backup")
 
 	var patchedLibraries []string
 
@@ -64,7 +65,7 @@ func JavaArchivePatchCommand(
 			}
 		}
 
-		err = patch.ProcessJavaArchive(finding, dryRun)
+		err = patch.ProcessJavaArchive(finding, dryRun, backup)
 		if err != nil {
 			log.Error().
 				Str("path", finding.Path).

tools/log4shell/constants/fs.go

@@ -21,3 +21,7 @@ const (
 	EarFileExt     = ".ear"
 	ClassFileExt   = ".class"
 )
+
+var (
+	CleanupDirs []string
+)

tools/log4shell/main.go

@@ -17,6 +17,7 @@ package main
 import (
 	"github.com/lunasec-io/lunasec/tools/log4shell/commands"
 	"github.com/lunasec-io/lunasec/tools/log4shell/constants"
+	"github.com/lunasec-io/lunasec/tools/log4shell/util"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/urfave/cli/v2"
@@ -48,6 +49,10 @@ func main() {
 
 	zerolog.SetGlobalLevel(zerolog.InfoLevel)
 
+	util.RunOnProcessExit(func() {
+		util.RemoveCleanupDirs()
+	})
+
 	globalBoolFlags := map[string]bool{
 		"verbose":         false,
 		"json":            false,
@@ -187,6 +192,10 @@ func main() {
 				Usage:   "Patches findings of libraries vulnerable toLog4Shell by removing the JndiLookup.class file from each.",
 				Before:  setGlobalBoolFlags,
 				Flags: []cli.Flag{
+					&cli.BoolFlag{
+						Name:  "backup",
+						Usage: "Backup each library to path/to/library.jar.bak before overwriting.",
+					},
 					&cli.StringSliceFlag{
 						Name:  "exclude",
 						Usage: "Exclude subdirectories from scanning. This can be helpful if there are directories which your user does not have access to when starting a scan from `/`.",

tools/log4shell/patch/archivepatch.go

@@ -332,7 +332,7 @@ func copyAndFilterFilesFromZip(
 	return
 }
 
-func ProcessJavaArchive(finding types.Finding, dryRun bool) (err error) {
+func ProcessJavaArchive(finding types.Finding, dryRun, backup bool) (err error) {
 	var (
 		libraryFile *os.File
 		zipReader   *zip.Reader
@@ -383,6 +383,23 @@ func ProcessJavaArchive(finding types.Finding, dryRun bool) (err error) {
 		return
 	}
 
+	if backup {
+		backupFilePath := fsFile + ".bak"
+		log.Info().
+			Str("libraryFileName", fsFile).
+			Str("backupFileName", backupFilePath).
+			Msg("Backing up library file before overwritting.")
+		_, err = util.CopyFile(fsFile, backupFilePath)
+		if err != nil {
+			log.Error().
+				Str("libraryFileName", fsFile).
+				Str("backupFileName", backupFilePath).
+				Err(err).
+				Msg("Unable to backup library file.")
+			return
+		}
+	}
+
 	_, err = util.CopyFile(filteredLibrary, fsFile)
 	if err != nil {
 		log.Error().

tools/log4shell/scan/executablejar.go

@@ -17,12 +17,13 @@ package scan
 import (
 	"bytes"
 	"github.com/lunasec-io/lunasec/tools/log4shell/constants"
+	"github.com/lunasec-io/lunasec/tools/log4shell/types"
 	"github.com/rs/zerolog/log"
 	"io"
 	"os"
 )
 
-func readerAtStartOfArchive(path string, file *os.File) (reader io.ReaderAt, offset int64, err error) {
+func readerAtStartOfArchive(path string, file *os.File) (reader types.ReaderAtCloser, offset int64, err error) {
 	// By default, we assume our original file will be our returned reader
 	reader = file
 
@@ -76,7 +77,7 @@ func readerAtStartOfArchive(path string, file *os.File) (reader io.ReaderAt, off
 				Msg("unable to locate start of archive in bash executable jar file")
 			return
 		}
-		reader = bytes.NewReader(fileContents[idx:])
+		reader = types.NopReaderAtCloser(bytes.NewReader(fileContents[idx:]))
 		offset = int64(idx)
 	}
 	return

tools/log4shell/scan/scan.go

@@ -21,7 +21,6 @@ import (
 	"github.com/lunasec-io/lunasec/tools/log4shell/types"
 	"github.com/lunasec-io/lunasec/tools/log4shell/util"
 	"github.com/rs/zerolog/log"
-	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -147,9 +146,65 @@ func (s *Log4jDirectoryScanner) scanLocatedArchive(
 	return s.scanArchiveForVulnerableFiles(path, reader, info.Size() - offset)
 }
 
+func (s *Log4jDirectoryScanner) getFilesToScan(
+	path string,
+	size int64,
+	zipReader *zip.Reader,
+) (filesToScan []types.FileToScan, cleanup func(), err error) {
+	if size > 1024 * 1024 * 1024 {
+		var (
+			tmpPath string
+			filenames []string
+		)
+
+		_, name := filepath.Split(path)
+		tmpPath, err = os.MkdirTemp(os.TempDir(), name)
+		if err != nil {
+			log.Warn().
+				Str("path", path).
+				Err(err).
+				Msg("unable to create temporary path")
+			return
+		}
+		util.EnsureDirIsCleanedUp(tmpPath)
+		cleanup = func() {
+			os.RemoveAll(tmpPath)
+			util.RemoveDirFromCleanup(tmpPath)
+		}
+
+		filenames, err = util.Unzip(zipReader, tmpPath)
+		if err != nil {
+			log.Warn().
+				Str("path", path).
+				Err(err).
+				Msg("unable to unzip file")
+			return
+		}
+
+		for _, file := range filenames {
+			dir, extractedFilename := filepath.Split(file)
+
+			fileToScan := &types.DiskFileToScan{
+				Filename: extractedFilename,
+				Path: dir,
+			}
+			filesToScan = append(filesToScan, fileToScan)
+		}
+		return
+	}
+
+	for _, zipFile := range zipReader.File {
+		fileToScan := &types.ZipFileToScan{
+			File: zipFile,
+		}
+		filesToScan = append(filesToScan, fileToScan)
+	}
+	return
+}
+
 func (s *Log4jDirectoryScanner) scanArchiveForVulnerableFiles(
 	path string,
-	reader io.ReaderAt,
+	reader types.ReaderAtCloser,
 	size int64,
 ) (findings []types.Finding) {
 	zipReader, err := zip.NewReader(reader, size)
@@ -161,31 +216,48 @@ func (s *Log4jDirectoryScanner) scanArchiveForVulnerableFiles(
 		return
 	}
 
-	for _, zipFile := range zipReader.File {
-		locatedFindings := s.scanFile(zipReader, path, zipFile)
+	filesToScan, cleanup, err := s.getFilesToScan(path, size, zipReader)
+	if err != nil {
+		return
+	}
+
+	resolveArchiveFile := util.ResolveZipFile(zipReader)
+
+	// if cleanup is specified, then we are reading files from disk
+	// and should close the current zip reader to free up space,
+	// set our archive reader to read files from disk, and defer
+	// a call to cleanup to remove all temporary extracted files
+	if cleanup != nil {
+		reader.Close()
+		resolveArchiveFile = util.ResolveDiskFile
+		defer cleanup()
+	}
+
+	for _, fileToScan := range filesToScan {
+		locatedFindings := s.scanFile(resolveArchiveFile, path, fileToScan)
 		findings = append(findings, locatedFindings...)
 	}
 	return
 }
 
 func (s *Log4jDirectoryScanner) scanFile(
-	zipReader *zip.Reader,
+	resolveArchiveFile types.ResolveArchiveFile,
 	path string,
-	file *zip.File,
+	file types.FileToScan,
 ) (findings []types.Finding) {
 	//log.Debug().
 	//	Str("path", path).
 	//	Str("file", file.Name).
 	//	Msg("Scanning archive file")
 
-	fileExt := util.FileExt(file.Name)
+	fileExt := util.FileExt(file.Name())
 	switch fileExt {
 	case constants.ClassFileExt:
 		if s.onlyScanArchives {
 			return
 		}
 
-		finding := s.scanArchiveFile(zipReader, path, file)
+		finding := s.scanArchiveFile(resolveArchiveFile, path, file)
 		if finding != nil {
 			findings = []types.Finding{*finding}
 		}
@@ -195,7 +267,7 @@ func (s *Log4jDirectoryScanner) scanFile(
 		 constants.ZipFileExt,
 		 constants.EarFileExt:
 		if s.onlyScanArchives {
-			finding := s.scanArchiveFile(zipReader, path, file)
+			finding := s.scanArchiveFile(resolveArchiveFile, path, file)
 			if finding != nil {
 				findings = []types.Finding{*finding}
 			}
@@ -207,32 +279,32 @@ func (s *Log4jDirectoryScanner) scanFile(
 }
 
 func (s *Log4jDirectoryScanner) scanArchiveFile(
-	zipReader *zip.Reader,
+	resolveArchiveFile types.ResolveArchiveFile,
 	path string,
-	file *zip.File,
+	file types.FileToScan,
 ) (finding *types.Finding) {
-	reader, err := file.Open()
+	reader, err := file.Reader()
 	if err != nil {
 		log.Warn().
-			Str("classFile", file.Name).
+			Str("classFile", file.Name()).
 			Str("path", path).
 			Err(err).
 			Msg("unable to open class file")
 		return
 	}
 	defer reader.Close()
 
-	return s.processArchiveFile(zipReader, reader, path, file.Name)
+	return s.processArchiveFile(resolveArchiveFile, reader, path, file.Name())
 }
 
 func (s *Log4jDirectoryScanner) scanEmbeddedArchive(
 	path string,
-	file *zip.File,
+	file types.FileToScan,
 ) (findings []types.Finding) {
-	reader, err := file.Open()
+	reader, err := file.Reader()
 	if err != nil {
 		log.Warn().
-			Str("classFile", file.Name).
+			Str("classFile", file.Name()).
 			Str("path", path).
 			Err(err).
 			Msg("unable to open embedded archive")
@@ -243,15 +315,16 @@ func (s *Log4jDirectoryScanner) scanEmbeddedArchive(
 	buffer, err := ioutil.ReadAll(reader)
 	if err != nil {
 		log.Warn().
-			Str("classFile", file.Name).
+			Str("classFile", file.Name()).
 			Str("path", path).
 			Err(err).
 			Msg("unable to read embedded archive")
 		return
 	}
+	reader.Close()
 
-	newPath := path + "::" + file.Name
-	archiveReader := bytes.NewReader(buffer)
+	newPath := path + "::" + file.Name()
+	archiveReader := types.NopReaderAtCloser(bytes.NewReader(buffer))
 	archiveSize := int64(len(buffer))
 
 	return s.scanArchiveForVulnerableFiles(newPath, archiveReader, archiveSize)

tools/log4shell/scan/scan_test.go

@@ -41,6 +41,7 @@ func createNewScanner() (scanner Log4jVulnerableDependencyScanner, err error) {
 }
 
 func BenchmarkScanningForVulnerablePackages(b *testing.B) {
+	return
 	b.ReportAllocs()
 
 	scanner, err := createNewScanner()
@@ -54,6 +55,22 @@ func BenchmarkScanningForVulnerablePackages(b *testing.B) {
 	fmt.Printf("Number of findings: %d\n", len(findings))
 }
 
+func BenchmarkScanningForLargeArchives(b *testing.B) {
+	b.ReportAllocs()
+
+	scanner, err := createNewScanner()
+	if err != nil {
+		b.Error(err)
+		return
+	}
+
+	for i := 0; i < 10; i++ {
+		findings := scanner.Scan([]string{"../test/large-archives"})
+
+		fmt.Printf("Number of findings: %d\n", len(findings))
+	}
+}
+
 func TestForFalsePositiveLibraryFindings(t *testing.T) {
 
 	scanner, err := createNewScanner()