Fallback to remote image scan

DEVELOPMENT.md

@@ -9,7 +9,7 @@ go run ./cmd/agent
 
 Start tilt on local kind cluster with mockapi backend which is located in ./tools/mockapi.
 ```
-API_URL=http://mockapi tilt up
+API_URL=http://mockapi IMAGE_SCAN_ENABLED=true tilt up
 ```
 
 ### Run E2E tests locally

blobscache/server.go

@@ -13,7 +13,7 @@ type ServerConfig struct {
 
 func NewServer(log logrus.FieldLogger, cfg ServerConfig) *Server {
 	return &Server{
-		log:        log,
+		log:        log.WithField("component", "blobscache"),
 		cfg:        cfg,
 		blobsCache: newMemoryBlobsCacheStore(log),
 	}

charts/castai-kvisor/values.yaml

@@ -83,7 +83,6 @@ config: |
     enabled: true
     scanInterval: "15s"
     maxConcurrentScans: 3
-    hostfsSocketFallbackEnabled: true
     apiUrl: "http://kvisor.{{ .Release.Namespace }}.svc.cluster.local.:6060"
     image:
       name: "{{ .Values.image.repository }}-imgcollector:{{ .Values.image.tag | default .Chart.AppVersion }}"

cmd/imgcollector/collector/collector.go

@@ -158,11 +158,11 @@ func (c *Collector) getImage(ctx context.Context) (image.ImageWithIndex, func(),
 	if c.cfg.Mode == config.ModeRemote {
 		opts := image.DockerOption{}
 		if c.cfg.DockerOptionPath != "" {
-			bytes, err := os.ReadFile(c.cfg.DockerOptionPath)
+			optsData, err := os.ReadFile(c.cfg.DockerOptionPath)
 			if err != nil {
 				return nil, nil, fmt.Errorf("reading docker options file: %w", err)
 			}
-			if err := yaml.Unmarshal(bytes, &opts); err != nil {
+			if err := yaml.Unmarshal(optsData, &opts); err != nil {
 				return nil, nil, fmt.Errorf("unmarshaling docker options file: %w", err)
 			}
 		}

cmd/imgcollector/collector/collector_test.go

@@ -27,7 +27,7 @@ import (
 func TestCollector(t *testing.T) {
 	t.Run("collect and sends metadata", func(t *testing.T) {
 		imgName := "notused"
-		imgID := "gke.gcr.io/phpmyadmin@sha256:b0d9c54760b35edd1854e5710c1a62a28ad2d2b070c801da3e30a3e59c19e7e3"
+		imgID := "gke.gcr.io/phpmyadmin@sha256:b0d9c54760b35edd1854e5710c1a62a28ad2d2b070c801da3e30a3e59c19e7e3" //nolint:gosec
 
 		r := require.New(t)
 		ctx := context.Background()

cmd/imgcollector/main.go

@@ -24,8 +24,9 @@ var (
 )
 
 func main() {
-	log := logrus.New()
-	log.SetLevel(logrus.DebugLevel)
+	logger := logrus.New()
+	logger.SetLevel(logrus.DebugLevel)
+	log := logger.WithField("component", "imagescan_job")
 	log.Infof("running image scan job, version=%s, commit=%s", Version, GitCommit)
 
 	cfg, err := config.FromEnv()

config/config.go

@@ -65,22 +65,21 @@ type CloudScanEKS struct {
 }
 
 type ImageScan struct {
-	Enabled                     bool           `envconfig:"IMAGE_SCAN_ENABLED" yaml:"enabled"`
-	ScanInterval                time.Duration  `envconfig:"IMAGE_SCAN_SCAN_INTERVAL" yaml:"scanInterval"`
-	ScanTimeout                 time.Duration  `envconfig:"IMAGE_SCAN_SCAN_TIMEOUT" yaml:"scanTimeout"`
-	MaxConcurrentScans          int64          `envconfig:"IMAGE_SCAN_MAX_CONCURRENT_SCANS" yaml:"maxConcurrentScans"`
-	Image                       ImageScanImage `envconfig:"IMAGE_SCAN_IMAGE" yaml:"image"`
-	Mode                        string         `envconfig:"IMAGE_SCAN_MODE" yaml:"mode"`
-	APIUrl                      string         `envconfig:"IMAGE_SCAN_API_URL" yaml:"apiUrl"`
-	HostfsSocketFallbackEnabled bool           `envconfig:"IMAGE_SCAN_HOSTFS_SOCKET_FALLBACK_ENABLED" yaml:"hostfsSocketFallbackEnabled"`
-	DockerOptionsPath           string         `envconfig:"IMAGE_SCAN_DOCKER_OPTIONS_PATH" yaml:"dockerOptionsPath"`
-	CPURequest                  string         `envconfig:"IMAGE_SCAN_CPU_REQUEST" yaml:"cpuRequest"`
-	CPULimit                    string         `envconfig:"IMAGE_SCAN_CPU_LIMIT" yaml:"cpuLimit"`
-	MemoryRequest               string         `envconfig:"IMAGE_SCAN_MEMORY_REQUEST" yaml:"memoryRequest"`
-	MemoryLimit                 string         `envconfig:"IMAGE_SCAN_MEMORY_LIMIT" yaml:"memoryLimit"`
-	Force                       bool           `envconfig:"IMAGE_SCAN_FORCE" yaml:"force"`
-	ProfileEnabled              bool           `envconfig:"IMAGE_SCAN_PROFILE_ENABLED" yaml:"profileEnabled"`
-	PhlareEnabled               bool           `envconfig:"IMAGE_SCAN_PHLARE_ENABLED" yaml:"phlareEnabled"`
+	Enabled            bool           `envconfig:"IMAGE_SCAN_ENABLED" yaml:"enabled"`
+	ScanInterval       time.Duration  `envconfig:"IMAGE_SCAN_SCAN_INTERVAL" yaml:"scanInterval"`
+	ScanTimeout        time.Duration  `envconfig:"IMAGE_SCAN_SCAN_TIMEOUT" yaml:"scanTimeout"`
+	MaxConcurrentScans int64          `envconfig:"IMAGE_SCAN_MAX_CONCURRENT_SCANS" yaml:"maxConcurrentScans"`
+	Image              ImageScanImage `envconfig:"IMAGE_SCAN_IMAGE" yaml:"image"`
+	Mode               string         `envconfig:"IMAGE_SCAN_MODE" yaml:"mode"`
+	APIUrl             string         `envconfig:"IMAGE_SCAN_API_URL" yaml:"apiUrl"`
+	DockerOptionsPath  string         `envconfig:"IMAGE_SCAN_DOCKER_OPTIONS_PATH" yaml:"dockerOptionsPath"`
+	CPURequest         string         `envconfig:"IMAGE_SCAN_CPU_REQUEST" yaml:"cpuRequest"`
+	CPULimit           string         `envconfig:"IMAGE_SCAN_CPU_LIMIT" yaml:"cpuLimit"`
+	MemoryRequest      string         `envconfig:"IMAGE_SCAN_MEMORY_REQUEST" yaml:"memoryRequest"`
+	MemoryLimit        string         `envconfig:"IMAGE_SCAN_MEMORY_LIMIT" yaml:"memoryLimit"`
+	Force              bool           `envconfig:"IMAGE_SCAN_FORCE" yaml:"force"`
+	ProfileEnabled     bool           `envconfig:"IMAGE_SCAN_PROFILE_ENABLED" yaml:"profileEnabled"`
+	PhlareEnabled      bool           `envconfig:"IMAGE_SCAN_PHLARE_ENABLED" yaml:"phlareEnabled"`
 }
 
 type ImageScanImage struct {
@@ -166,9 +165,6 @@ func Load(configPath string) (Config, error) {
 		if cfg.ImageScan.Image.PullPolicy == "" {
 			cfg.ImageScan.Image.PullPolicy = "IfNotPresent"
 		}
-		if cfg.ImageScan.DockerOptionsPath == "" {
-			cfg.ImageScan.DockerOptionsPath = "/etc/docker/config.json"
-		}
 		if cfg.ImageScan.MaxConcurrentScans == 0 {
 			cfg.ImageScan.MaxConcurrentScans = 3
 		}

config/config_test.go

@@ -76,14 +76,13 @@ func newTestConfig() Config {
 				Name:       "collector-img",
 				PullPolicy: "IfNotPresent",
 			},
-			Mode:                        "mode",
-			HostfsSocketFallbackEnabled: true,
-			DockerOptionsPath:           "/etc/config/docker-config.json",
-			CPURequest:                  "100m",
-			CPULimit:                    "2",
-			MemoryRequest:               "100Mi",
-			MemoryLimit:                 "2Gi",
-			APIUrl:                      "http://kvisor.castai-agent.svc.cluster.local.:6060",
+			Mode:              "mode",
+			DockerOptionsPath: "/etc/config/docker-config.json",
+			CPURequest:        "100m",
+			CPULimit:          "2",
+			MemoryRequest:     "100Mi",
+			MemoryLimit:       "2Gi",
+			APIUrl:            "http://kvisor.castai-agent.svc.cluster.local.:6060",
 		},
 		Linter: Linter{
 			Enabled: true,

delta/subscriber.go

@@ -59,7 +59,7 @@ func NewSubscriber(
 		cancel:          cancel,
 		cfg:             cfg,
 		k8sVersionMinor: k8sVersionMinor,
-		log:             log,
+		log:             log.WithField("component", "delta"),
 		client:          client,
 		delta:           newDelta(log, logLevel, stateProvider),
 	}

imagescan/delta.go

@@ -74,6 +74,8 @@ type deltaState struct {
 	rs     map[string]*appsv1.ReplicaSet
 	jobs   map[string]*batchv1.Job
 	nodes  map[string]*node
+
+	hostFSDisabled bool
 }
 
 func (d *deltaState) Observe(response *castai.TelemetryResponse) {
@@ -321,8 +323,12 @@ func (d *deltaState) setImageScanError(i *image, err error) {
 
 	img.failures++
 	img.lastScanErr = err
-	if strings.Contains(err.Error(), "no such file or directory") {
+	if strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "failed to get the layer") {
 		img.lastScanErr = errImageScanLayerNotFound
+		d.hostFSDisabled = true
+	} else if strings.Contains(err.Error(), "UNAUTHORIZED") || strings.Contains(err.Error(), "MANIFEST_UNKNOWN") || strings.Contains(err.Error(), "DENIED") {
+		// Error codes from https://github.com/google/go-containerregistry/blob/190ad0e4d556f199a07951d55124f8a394ebccd9/pkg/v1/remote/transport/error.go#L115
+		img.lastScanErr = errPrivateImage
 	}
 
 	img.nextScan = time.Now().UTC().Add(img.retryBackoff.Step())
@@ -361,6 +367,13 @@ func (d *deltaState) nodeCount() int {
 	return len(d.nodes)
 }
 
+func (d *deltaState) isHostFsDisabled() bool {
+	d.mu.RLock()
+	defer d.mu.RUnlock()
+
+	return d.hostFSDisabled
+}
+
 func getContainerRuntime(containerID string) imgcollectorconfig.Runtime {
 	parts := strings.Split(containerID, "://")
 	if len(parts) != 2 {
@@ -469,7 +482,10 @@ type imageOwner struct {
 	podIDs map[string]struct{}
 }
 
-var errImageScanLayerNotFound = errors.New("image layer not found")
+var (
+	errImageScanLayerNotFound = errors.New("image layer not found")
+	errPrivateImage           = errors.New("private image")
+)
 
 type image struct {
 	id               string

imagescan/subscriber.go

@@ -35,7 +35,7 @@ func NewSubscriber(
 		cancel:          cancel,
 		imageScanner:    imageScanner,
 		delta:           delta,
-		log:             log,
+		log:             log.WithField("component", "imagescan"),
 		cfg:             cfg,
 		k8sVersionMinor: k8sVersionMinor,
 		timeGetter:      timeGetter(),
@@ -108,9 +108,15 @@ func (s *Subscriber) handleDelta(event controller.Event, o controller.Object) {
 func (s *Subscriber) scheduleScans(ctx context.Context) (rerr error) {
 	images := s.delta.getImages()
 	now := s.timeGetter()
+	var privateImagesCount int
 	pendingImages := lo.Filter(images, func(v *image, _ int) bool {
+		isPrivateImage := errors.Is(v.lastScanErr, errPrivateImage)
+		if isPrivateImage {
+			privateImagesCount++
+		}
 		return !v.scanned &&
 			len(v.owners) > 0 &&
+			!isPrivateImage &&
 			(v.nextScan.IsZero() || v.nextScan.Before(now))
 	})
 	sort.Slice(pendingImages, func(i, j int) bool {
@@ -120,6 +126,10 @@ func (s *Subscriber) scheduleScans(ctx context.Context) (rerr error) {
 	metrics.SetTotalImagesCount(len(images))
 	metrics.SetPendingImagesCount(len(pendingImages))
 
+	if privateImagesCount > 0 {
+		s.log.Warnf("skipping %d private images", privateImagesCount)
+	}
+
 	concurrentScans := s.concurrentScansNumber()
 	imagesForScan := pendingImages
 	if len(imagesForScan) > concurrentScans {
@@ -211,14 +221,7 @@ func (s *Subscriber) scanImage(ctx context.Context, img *image) (rerr error) {
 		metrics.ObserveScanDuration(metrics.ScanTypeImage, start)
 	}()
 
-	// If image scan fails in containerd hostfs mode due missing layers
-	// try to scan via mounted containerd socket.
-	mode := s.cfg.Mode
-	if img.lastScanErr != nil && errors.Is(img.lastScanErr, errImageScanLayerNotFound) &&
-		img.containerRuntime == imgcollectorconfig.RuntimeContainerd &&
-		s.cfg.HostfsSocketFallbackEnabled {
-		mode = string(imgcollectorconfig.ModeDaemon)
-	}
+	mode := s.getScanMode(img)
 
 	return s.imageScanner.ScanImage(ctx, ScanImageParams{
 		ImageName:                   img.name,
@@ -240,3 +243,16 @@ func (s *Subscriber) concurrentScansNumber() int {
 
 	return int(s.cfg.MaxConcurrentScans)
 }
+
+// getScanMode returns configured image scan mode if set.
+// If mode is empty it will be determined automatically based on container runtime inside scanner.go
+//
+// Special case:
+// If hostfs mode is used and image scan fails due to missing layers remote image scan will be used as fallback.
+func (s *Subscriber) getScanMode(img *image) string {
+	mode := s.cfg.Mode
+	if s.delta.isHostFsDisabled() || (img.lastScanErr != nil && errors.Is(img.lastScanErr, errImageScanLayerNotFound)) {
+		mode = string(imgcollectorconfig.ModeRemote)
+	}
+	return mode
+}

imagescan/subscriber_test.go

@@ -275,19 +275,18 @@ func TestSubscriber(t *testing.T) {
 		r.True(img.scanned)
 	})
 
-	t.Run("scan image with containerd sock fallback", func(t *testing.T) {
+	t.Run("scan image with remove mode fallback", func(t *testing.T) {
 		r := require.New(t)
 
 		cfg := config.ImageScan{
-			ScanInterval:                1 * time.Millisecond,
-			ScanTimeout:                 time.Minute,
-			MaxConcurrentScans:          5,
-			Mode:                        string(imgcollectorconfig.ModeHostFS),
-			HostfsSocketFallbackEnabled: true,
-			CPURequest:                  "500m",
-			CPULimit:                    "2",
-			MemoryRequest:               "100Mi",
-			MemoryLimit:                 "2Gi",
+			ScanInterval:       1 * time.Millisecond,
+			ScanTimeout:        time.Minute,
+			MaxConcurrentScans: 5,
+			Mode:               string(imgcollectorconfig.ModeHostFS),
+			CPURequest:         "500m",
+			CPULimit:           "2",
+			MemoryRequest:      "100Mi",
+			MemoryLimit:        "2Gi",
 		}
 
 		scanner := &mockImageScanner{}
@@ -323,7 +322,7 @@ func TestSubscriber(t *testing.T) {
 		err := sub.scheduleScans(ctx)
 		r.NoError(err)
 		r.Len(scanner.imgs, 1)
-		r.Equal(string(imgcollectorconfig.ModeDaemon), scanner.imgs[0].Mode)
+		r.Equal(string(imgcollectorconfig.ModeRemote), scanner.imgs[0].Mode)
 	})
 
 	t.Run("respect node count", func(t *testing.T) {

linters/kubelinter/subscriber.go

@@ -4,11 +4,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	batchv1 "k8s.io/api/batch/v1"
 	"reflect"
 	"strings"
 	"time"
 
+	batchv1 "k8s.io/api/batch/v1"
+
 	"github.com/samber/lo"
 	"github.com/sirupsen/logrus"
 	"golang.stackrox.io/kube-linter/pkg/lintcontext"