fix(megaraid): rescan SCSI bus after creating a virtual drive

pkg/implementation/raidcontroller/megaraid/logicalvolume.go

@@ -2,10 +2,12 @@ package megaraid
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 
@@ -15,11 +17,38 @@ import (
 	"github.com/scality/raidmgmt/pkg/utils"
 )
 
-// patternLV is the pattern for the logical volume selector.
 const (
+	// patternLV is the pattern for the logical volume selector.
 	patternLV string = "/c%d/v%s"
+
+	// scsiHostRoot holds one sysfs directory per SCSI host adapter.
+	scsiHostRoot = "/sys/class/scsi_host"
+
+	// megaraidSASDriver is the proc_name of the MegaRAID kernel driver, used to
+	// select which SCSI hosts to rescan for newly created virtual drives.
+	megaraidSASDriver = "megaraid_sas"
+)
+
+// NewVolumeSettleTimeout and NewVolumeSettleInterval bound the wait for a
+// freshly created volume's device node to appear after "add vd". The kernel
+// only auto-discovers the first few VDs created in rapid succession (the
+// megaraid_sas driver stops emitting hotplug events), so createLV polls until
+// the new volume resolves, forcing a bus rescan between failed attempts.
+// Exported so tests can shrink them (like CustomRescanSCSIHosts).
+//
+//nolint:gochecknoglobals // Tunables for the post-create device-settle poll.
+var (
+	NewVolumeSettleTimeout  = 60 * time.Second
+	NewVolumeSettleInterval = 1 * time.Second
 )
 
+// CustomRescanSCSIHosts forces the kernel to probe for newly created virtual
+// drives. It is a package-level var so tests can stub the sysfs side effect,
+// mirroring CustomFileExists / CustomEvalSymlinks.
+//
+//nolint:gochecknoglobals // Test seam for the sysfs bus rescan.
+var CustomRescanSCSIHosts = rescanMegaraidHosts
+
 // logicalvolumes returns all logical volumes for a given controller.
 func (a *Adapter) logicalvolumes(metadata *raidcontroller.Metadata) (
 	[]*logicalvolume.LogicalVolume,
@@ -315,15 +344,94 @@ func (a *Adapter) createLV(request *logicalvolume.Request) (
 		return nil, errors.Wrap(err, ErrCommandFailed.Error())
 	}
 
-	// Get the newly created logical volume
-	newLV, err := a.findNewLogicalVolume(request.PDrivesMetadata)
+	// Get the newly created logical volume. storcli returns before the kernel
+	// has a block device for the new volume, so poll until it resolves (forcing
+	// a bus rescan between attempts) instead of failing on the first attempt.
+	newLV, err := a.findNewVolumeUntilSettled(request.PDrivesMetadata)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to find new logical volume")
 	}
 
 	return newLV, nil
 }
 
+// findNewVolumeUntilSettled retries findNewLogicalVolume until the newly created
+// volume's device node has been discovered and its permanent path resolves, or
+// NewVolumeSettleTimeout elapses. When a lookup fails it forces a SCSI bus
+// rescan before retrying: the megaraid_sas driver stops auto-announcing VDs
+// after a few rapid creations, leaving later volumes with no /dev node until the
+// kernel is told to probe the bus. Volumes the kernel discovers on its own
+// resolve on the first attempt with no rescan.
+func (a *Adapter) findNewVolumeUntilSettled(pds []*physicaldrive.Metadata) (
+	*logicalvolume.LogicalVolume,
+	error,
+) {
+	deadline := time.Now().Add(NewVolumeSettleTimeout)
+
+	var lastRescanErr error
+
+	for {
+		lv, err := a.findNewLogicalVolume(pds)
+		if err == nil {
+			return lv, nil
+		}
+
+		if time.Now().After(deadline) {
+			if lastRescanErr != nil {
+				return nil, errors.Wrapf(err,
+					"device node not settled after %s (scsi rescan failing: %v)",
+					NewVolumeSettleTimeout, lastRescanErr)
+			}
+
+			return nil, errors.Wrapf(err, "device node not settled after %s",
+				NewVolumeSettleTimeout)
+		}
+
+		// The volume did not resolve: the kernel may not have discovered its
+		// device node yet. Force a bus rescan, then wait before retrying. A
+		// rescan failure is not fatal on its own, but it is surfaced on timeout.
+		if rescanErr := CustomRescanSCSIHosts(); rescanErr != nil {
+			lastRescanErr = rescanErr
+		}
+
+		time.Sleep(NewVolumeSettleInterval)
+	}
+}
+
+// rescanMegaraidHosts asks the kernel to probe the SCSI bus for newly created
+// virtual drives on the MegaRAID controller. Only megaraid_sas hosts are
+// scanned, to avoid the multi-second link-down timeouts of empty SATA ports.
+func rescanMegaraidHosts() error {
+	entries, err := os.ReadDir(scsiHostRoot)
+	if err != nil {
+		return errors.Wrap(err, "failed to list scsi hosts")
+	}
+
+	scanned := 0
+
+	for _, entry := range entries {
+		hostDir := filepath.Join(scsiHostRoot, entry.Name())
+
+		procName, err := os.ReadFile(filepath.Join(hostDir, "proc_name"))
+		if err != nil || strings.TrimSpace(string(procName)) != megaraidSASDriver {
+			continue
+		}
+
+		// "- - -" means "scan every channel, target and LUN" on this host.
+		if err := os.WriteFile(filepath.Join(hostDir, "scan"), []byte("- - -"), 0); err != nil {
+			return errors.Wrapf(err, "failed to rescan scsi host %s", entry.Name())
+		}
+
+		scanned++
+	}
+
+	if scanned == 0 {
+		return errors.New("no megaraid_sas scsi host found to rescan")
+	}
+
+	return nil
+}
+
 // megaraidCreateCacheFlags returns the "add vd" cache flags. Each policy is
 // mapped to its megaraid CLI token through the adapter-owned mappers rather than
 // emitted from the domain enum's string value (see cache.go). The megaraid v1

pkg/implementation/raidcontroller/megaraid/megaraid_test.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/pkg/errors"
 
@@ -49,6 +50,10 @@ func (s *UnitTestSuite) SetupTest() {
 	s.a = megaraid2.New(s.mockRunner)
 
 	s.mockPathResolver = mocks2.NewPathResolver(s.T())
+
+	// Never touch the real /sys during unit tests; createLV triggers a SCSI
+	// bus rescan, which tests override individually when they assert on it.
+	megaraid2.CustomRescanSCSIHosts = func() error { return nil }
 }
 
 // mockOutput reads the output from a file and returns it.
@@ -708,6 +713,78 @@ func (s *UnitTestSuite) TestCreateLV() {
 	}
 }
 
+// TestCreateLVForcesRescanAndSettles reproduces the post-create discovery race:
+// storcli returns before the kernel has a block device for the new volume, so
+// the first path resolution fails ("failed to compute permanent path") while a
+// later one succeeds. createLV must force a SCSI bus rescan and poll until the
+// device node is discovered instead of failing on the first try.
+func (s *UnitTestSuite) TestCreateLVForcesRescanAndSettles() {
+	s.wasCreateLVCalledOnce = false
+
+	// Shrink the settle poll so the retry does not add real wall-clock time.
+	origTimeout := megaraid2.NewVolumeSettleTimeout
+	origInterval := megaraid2.NewVolumeSettleInterval
+	megaraid2.NewVolumeSettleTimeout = 5 * time.Second
+	megaraid2.NewVolumeSettleInterval = time.Millisecond
+
+	defer func() {
+		megaraid2.NewVolumeSettleTimeout = origTimeout
+		megaraid2.NewVolumeSettleInterval = origInterval
+	}()
+
+	s.setupMockCallsCreateLV()
+
+	// Stub the sysfs bus rescan and count how often it runs (never touch the
+	// real /sys in unit tests).
+	rescanCalls := 0
+	origRescan := megaraid2.CustomRescanSCSIHosts
+	megaraid2.CustomRescanSCSIHosts = func() error {
+		rescanCalls++
+
+		return nil
+	}
+
+	defer func() { megaraid2.CustomRescanSCSIHosts = origRescan }()
+
+	// First resolution: device not yet discovered. For the single-drive volume
+	// this falls back to ComputePaths on the real filesystem, which fails. Every
+	// subsequent resolution finds the symlink.
+	s.mockPathResolver.On("FileExists", mock.Anything).Return(false).Once()
+	s.mockPathResolver.On("FileExists", mock.Anything).Return(true)
+	s.mockPathResolver.On(
+		"EvalSymlinks",
+		"/dev/disk/by-id/wwn-0x600062b212da5d402bd3b493e1699377",
+	).Return("/dev/sda", nil)
+
+	s.setupCustomFileExists()
+	defer s.restoreCustomFileExists()
+
+	s.setupCustomEvalSymlinks()
+	defer s.restoreCustomEvalSymlinks()
+
+	request := &logicalvolume.Request{
+		CtrlMetadata: &raidcontroller.Metadata{ID: 0},
+		RAIDLevel:    logicalvolume.RAIDLevel0,
+		PDrivesMetadata: []*physicaldrive.Metadata{
+			{CtrlMetadata: &raidcontroller.Metadata{ID: 0}, ID: "251:12"},
+		},
+		CacheOptions: &logicalvolume.CacheOptions{
+			ReadPolicy:  logicalvolume.ReadPolicyReadAhead,
+			WritePolicy: logicalvolume.WritePolicyWriteThrough,
+			IOPolicy:    logicalvolume.IOPolicyDirect,
+		},
+	}
+
+	newLv, err := s.a.CreateLV(request)
+
+	s.Require().NoError(err)
+	s.Require().NotNil(newLv)
+	s.Equal("228", newLv.ID)
+	s.Equal("/dev/sda", newLv.DevicePath)
+	s.Equal("/dev/disk/by-id/wwn-0x600062b212da5d402bd3b493e1699377", newLv.PermanentPath)
+	s.GreaterOrEqual(rescanCalls, 1, "createLV must force a SCSI rescan for the new volume")
+}
+
 func (s *UnitTestSuite) TestDeleteLV() {
 	s.mockRunner.On("Run", []string{"/c0/v228", "delete"}).
 		Return(mockReturn("logicalvolumes/delete/success"))