diff --git a/virtcontainers/api.go b/virtcontainers/api.go
index 5a1f37aed1..3b147886a1 100644
--- a/virtcontainers/api.go
+++ b/virtcontainers/api.go
@@ -102,6 +102,10 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f
 
 	// Move runtime to sandbox cgroup so all process are created there.
 	if s.config.SandboxCgroupOnly {
+		if err := s.createCgroupManager(); err != nil {
+			return nil, err
+		}
+
 		if err := s.setupSandboxCgroup(); err != nil {
 			return nil, err
 		}
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index cdba81a1dc..353ad80480 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -606,10 +606,6 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 		}
 	}
 
-	if err := s.createCgroupManager(); err != nil {
-		return nil, err
-	}
-
 	agentConfig, err := newAgentConfig(sandboxConfig.AgentType, sandboxConfig.AgentConfig)
 	if err != nil {
 		return nil, err
@@ -739,6 +735,12 @@ func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err
 		return nil, fmt.Errorf("failed to create sandbox with config %+v: %v", config, err)
 	}
 
+	if sandbox.config.SandboxCgroupOnly {
+		if err := sandbox.createCgroupManager(); err != nil {
+			return nil, err
+		}
+	}
+
 	// This sandbox already exists, we don't need to recreate the containers in the guest.
 	// We only need to fetch the containers from storage and create the container structs.
 	if err := sandbox.fetchContainers(); err != nil {