diff --git a/pkg/cmd/factory.go b/pkg/cmd/factory.go
index de28a9fb0..5a54ce4f6 100644
--- a/pkg/cmd/factory.go
+++ b/pkg/cmd/factory.go
@@ -99,3 +99,40 @@ func NewDefaultFactory(cfg *rest.Config) Factory {
 	copiedCfg.Wrap(multicluster.NewSecretModeMultiClusterRoundTripper)
 	return &defaultFactory{cfg: &copiedCfg}
 }
+
+type deferredFactory struct {
+	sync.Mutex
+	Factory
+	ConfigGetter
+}
+
+// NewDeferredFactory create a factory that will only get KubeConfig until it is needed for the first time
+func NewDeferredFactory(getter ConfigGetter) Factory {
+	return &deferredFactory{ConfigGetter: getter}
+}
+
+func (f *deferredFactory) init() {
+	cfg, err := f.ConfigGetter()
+	cmdutil.CheckErr(err)
+	f.Factory = NewDefaultFactory(cfg)
+}
+
+// Config return the kubeConfig
+func (f *deferredFactory) Config() *rest.Config {
+	f.Lock()
+	defer f.Unlock()
+	if f.Factory == nil {
+		f.init()
+	}
+	return f.Factory.Config()
+}
+
+// Client return the kubeClient
+func (f *deferredFactory) Client() client.Client {
+	f.Lock()
+	defer f.Unlock()
+	if f.Factory == nil {
+		f.init()
+	}
+	return f.Factory.Client()
+}
diff --git a/references/cli/cli.go b/references/cli/cli.go
index d05d2bdf9..ec6f05d8a 100644
--- a/references/cli/cli.go
+++ b/references/cli/cli.go
@@ -71,8 +71,9 @@ func NewCommandWithIOStreams(ioStream util.IOStreams) *cobra.Command {
 	commandArgs := common.Args{
 		Schema: common.Scheme,
 	}
-	f := velacmd.NewDefaultFactory(config.GetConfigOrDie())
+	f := velacmd.NewDeferredFactory(config.GetConfig)
 
+	_, _ = commandArgs, f
 	if err := system.InitDirs(); err != nil {
 		fmt.Println("InitDir err", err)
 		os.Exit(1)