refactor Driver interface

This commit is contained in:
Josh Wolf
2021-06-15 15:13:18 -06:00
parent fe6060af3c
commit e374314cae
20 changed files with 2006 additions and 307 deletions

View File

@@ -1,28 +1,4 @@
# Hauler - Kubernetes Air Gap Migration
```bash
# _ _
# | |__ __ _ _ _| | ___ _ __
# | '_ \ / _` | | | | |/ _ \ '__|
# | | | | (_| | |_| | | __/ |
# |_| |_|\__,_|\__,_|_|\___|_|
#
# , , _______________________________
# ,-----------|'------'| | |
# /. '-' |-' |_____________________________|
# |/| | |
# | .________.'----' _______________________________
# | || | || | |
# \__|' \__|' |_____________________________|
#
# __________________________________________________________
# | |
# |________________________________________________________|
#
# __________________________________________________________
# | |
# |________________________________________________________|
```
## WARNING - Work In Progress
@@ -37,15 +13,41 @@ Kubernetes-focused software usually relies on executables, archives, container i
Hauler aims to fill this gap by standardizing low-level components of this stack and automating the collection and transfer of artifacts.
## Usage
Package a self contained deployable cluster
```bash
# CLI
# bare k3s cluster
hauler create
# k3s cluster with autodeployed manifests on boot
hauler create -p path/to/rawmanifests -p path/to/kustomizebase -p path/to/helmchart -i image
# Config File
hauler create [-c ./package.yaml]
```
Bootstrap a cluster from a packaged archive
```bash
hauler boot package.tar.zst
```
Relocate a set of images
```bash
hauler save -p path/to/manifests -i image:tag -i image@sha256:...
hauler relocate bundle.tar.zst airgap-registry:5000
```
## Additional Details
- [Roadmap](./ROADMAP.md)
- [Vagrant](./VAGRANT.md)
## Go CLI
The initial MVP for a hauler CLI used to streamline the packaging and deploying processes is in the `cmd/` and `pkg/` folders, along with `go.mod` and `go.sum`. Currently only a `package` subcommand is supported, which generates a `.tar.gz` archive used in the future `deploy` subcommand.
### Build
To build hauler, the Go CLI v1.14 or higher is required. See <https://golang.org/dl/> for downloads and see <https://golang.org/doc/install> for installation instructions.

View File

@@ -2,14 +2,11 @@ package app
import (
"context"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
"github.com/rancherfederal/hauler/pkg/bootstrap"
"github.com/rancherfederal/hauler/pkg/driver"
"github.com/rancherfederal/hauler/pkg/packager"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"os"
"path/filepath"
"sigs.k8s.io/yaml"
)
type deployOpts struct {
@@ -62,44 +59,28 @@ func (o *deployOpts) Run(packagePath string) error {
return err
}
bundleData, err := os.ReadFile(filepath.Join(tmpdir, "package.json"))
if err != nil {
return err
}
var p v1alpha1.Package
if err := yaml.Unmarshal(bundleData, &p); err != nil {
return err
}
d := v1alpha1.NewDriver(p.Spec.Driver.Kind)
bootLogger := o.logger.WithFields(logrus.Fields{
"driver": p.Spec.Driver.Kind,
})
b, err := bootstrap.NewBooter(tmpdir)
if err != nil {
return err
}
o.logger.Infof("Initializing package for driver: %s", p.Spec.Driver.Kind)
if err := b.Init(); err != nil {
d := driver.NewDriver(b.Package.Spec.Driver)
if err != nil {
return err
}
o.logger.Infof("Performing pre %s boot steps", p.Spec.Driver.Kind)
if err := b.PreBoot(ctx, d, bootLogger); err != nil {
o.logger.Infof("Performing pre %s boot steps", b.Package.Spec.Driver.Type)
if err := b.PreBoot(ctx, d, o.logger); err != nil {
return err
}
o.logger.Infof("Booting %s", p.Spec.Driver.Kind)
if err := b.Boot(ctx, d, bootLogger); err != nil {
o.logger.Infof("Booting %s", b.Package.Spec.Driver.Type)
if err := b.Boot(ctx, d, o.logger); err != nil {
return err
}
o.logger.Infof("Performing post %s boot steps", p.Spec.Driver.Kind)
if err := b.PostBoot(ctx, d, bootLogger); err != nil {
o.logger.Infof("Performing post %s boot steps", b.Package.Spec.Driver.Type)
if err := b.PostBoot(ctx, d, o.logger); err != nil {
return err
}

View File

@@ -3,6 +3,7 @@ package app
import (
"context"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
"github.com/rancherfederal/hauler/pkg/driver"
"github.com/rancherfederal/hauler/pkg/packager"
"github.com/spf13/cobra"
"os"
@@ -41,7 +42,7 @@ Container images, git repositories, and more, packaged and ready to be served wi
f := cmd.Flags()
f.StringVarP(&opts.driver, "driver", "d", "k3s",
"Driver type to use for package (k3s or rke2)")
"IDriver type to use for package (k3s or rke2)")
f.StringVarP(&opts.outputFile, "output", "o", "haul.tar.zst",
"package output location relative to the current directory (haul.tar.zst)")
f.StringVarP(&opts.configFile, "config", "c", "./package.yaml",
@@ -82,21 +83,21 @@ func (o *createOpts) Run() error {
}
defer os.RemoveAll(tmpdir)
pkgr := packager.NewPackager(tmpdir)
pkgr := packager.NewPackager(tmpdir, o.logger)
o.logger.Infof("Packaging driver (%s %s) artifacts...", p.Spec.Driver.Version, p.Spec.Driver.Kind)
d := v1alpha1.NewDriver(p.Spec.Driver.Kind)
if err = pkgr.Driver(ctx, d); err != nil {
o.logger.Infof("Packaging driver (%s %s) artifacts...", p.Spec.Driver.Version, p.Spec.Driver.Type)
d := driver.NewDriver(p.Spec.Driver)
if err = pkgr.PackageDriver(ctx, d); err != nil {
return err
}
o.logger.Infof("Packaging fleet artifacts...")
if err = pkgr.Fleet(ctx, p.Spec.Fleet); err != nil {
if err = pkgr.PackageFleet(ctx, p.Spec.Fleet); err != nil {
return err
}
o.logger.Infof("Packaging images and manifests defined in specified paths...")
if _, err = pkgr.Bundles(ctx, p.Spec.Paths...); err != nil {
if _, err = pkgr.PackageBundles(ctx, p.Spec.Paths...); err != nil {
return err
}

View File

@@ -24,7 +24,7 @@ func Test_createOpts_Run(t *testing.T) {
Spec: v1alpha1.PackageSpec{
Fleet: v1alpha1.Fleet{Version: "0.3.5"},
Driver: v1alpha1.Driver{
Kind: "k3s",
Type: "k3s",
Version: "v1.21.1+k3s1",
},
Paths: []string{

26
cmd/haulerctl/app/pkg.go Normal file
View File

@@ -0,0 +1,26 @@
package app
import "github.com/spf13/cobra"
type pkgOpts struct{}
func NewPkgCommand() *cobra.Command {
opts := &pkgOpts{}
//TODO
_ = opts
cmd := &cobra.Command{
Use: "pkg",
Short: "",
Long: "",
Aliases: []string{"p", "package"},
RunE: func(cmd *cobra.Command, args []string) error {
return cmd.Help()
},
}
cmd.AddCommand(NewPkgCreateCommand())
cmd.AddCommand(NewPkgBootCommand())
return cmd
}

View File

@@ -0,0 +1,34 @@
package app
import "github.com/spf13/cobra"
type pkgBootOpts struct {
cfgFile string
}
func NewPkgBootCommand() *cobra.Command {
opts := pkgBootOpts{}
cmd := &cobra.Command{
Use: "boot",
Short: "",
Long: "",
Aliases: []string{"b", "bootstrap"},
PreRunE: func(cmd *cobra.Command, args []string) error {
return opts.PreRun()
},
RunE: func(cmd *cobra.Command, args []string) error {
return opts.Run()
},
}
return cmd
}
func (o *pkgBootOpts) PreRun() error {
return nil
}
func (o *pkgBootOpts) Run() error {
return nil
}

View File

@@ -0,0 +1,106 @@
package app
import (
"context"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"os"
"sigs.k8s.io/yaml"
)
type pkgCreateOpts struct {
cfgFile string
driver string
driverVersion string
fleetVersion string
images []string
paths []string
}
func NewPkgCreateCommand() *cobra.Command {
opts := pkgCreateOpts{}
cmd := &cobra.Command{
Use: "create",
Short: "",
Long: "",
Aliases: []string{"c"},
PreRunE: func(cmd *cobra.Command, args []string) error {
return opts.PreRun()
},
RunE: func(cmd *cobra.Command, args []string) error {
return opts.Run()
},
}
f := cmd.PersistentFlags()
f.StringVarP(&opts.cfgFile, "config", "c", "./pkg.yaml",
"path to config file")
f.StringVarP(&opts.driver, "driver", "d", "k3s",
"")
f.StringVar(&opts.driverVersion, "driver-version", "v1.21.1+k3s1",
"")
f.StringVar(&opts.fleetVersion, "fleet-version", "v0.3.5",
"")
f.StringSliceVarP(&opts.images, "image", "i", []string{},
"")
f.StringSliceVarP(&opts.paths, "path", "p", []string{},
"")
return cmd
}
func (o *pkgCreateOpts) PreRun() error {
_, err := os.Stat(o.cfgFile)
if os.IsNotExist(err) {
logrus.Infof("Could not find %s, creating one", o.cfgFile)
p := v1alpha1.Package{
TypeMeta: metav1.TypeMeta{
Kind: "",
APIVersion: "",
},
ObjectMeta: metav1.ObjectMeta{
Name: "",
},
Spec: v1alpha1.PackageSpec{
Fleet: v1alpha1.Fleet{
Version: o.fleetVersion,
},
Driver: v1alpha1.Driver{
Type: o.driver,
Version: o.driverVersion,
},
Paths: o.paths,
Images: o.images,
},
}
data, err := yaml.Marshal(p)
if err != nil {
return err
}
if err := os.WriteFile(o.cfgFile, data, 0644); err != nil {
return err
}
}
if err != nil {
return err
}
return nil
}
func (o *pkgCreateOpts) Run() error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
_ = ctx
return nil
}

View File

@@ -67,9 +67,11 @@ func NewRootCommand() *cobra.Command {
cobra.OnInitialize(initConfig)
cmd.AddCommand(NewRelocateCommand())
cmd.AddCommand(NewCreateCommand())
cmd.AddCommand(NewBundleCommand())
cmd.AddCommand(NewCopyCommand())
cmd.AddCommand(NewPkgCommand())
cmd.AddCommand(NewCreateCommand())
cmd.AddCommand(NewBootstrapCommand())
f := cmd.PersistentFlags()

3
go.mod
View File

@@ -17,7 +17,7 @@ require (
github.com/imdario/mergo v0.3.12
github.com/klauspost/compress v1.13.0 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/mattn/go-runewidth v0.0.12 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/mholt/archiver/v3 v3.5.0
github.com/mitchellh/go-homedir v1.1.0
github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6
@@ -26,7 +26,6 @@ require (
github.com/pelletier/go-toml v1.8.1 // indirect
github.com/rancher/fleet v0.3.5
github.com/rancher/fleet/pkg/apis v0.0.0
github.com/rivo/uniseg v0.2.0 // indirect
github.com/sirupsen/logrus v1.8.1
github.com/spf13/afero v1.6.0
github.com/spf13/cobra v1.1.3

5
go.sum
View File

@@ -926,8 +926,8 @@ github.com/mattn/go-oci8 v0.0.7/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mN
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow=
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.12.0 h1:u/x3mp++qUxvYfulZ4HKOvVO0JWhk7HtE8lWhbGz/Do=
@@ -1192,7 +1192,6 @@ github.com/rancher/wrangler v0.7.3-0.20210224225730-5ed69efb6ab9/go.mod h1:9du9G
github.com/rancher/wrangler-cli v0.0.0-20200815040857-81c48cf8ab43/go.mod h1:KxpGNhk/oVL6LCfyxESTD1sb8eXRlUxtkbNm06+7dZU=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/robfig/cron v1.1.0 h1:jk4/Hud3TTdcrJgUOBgsqrZBarcxl6ADIjSC2iniwLY=

View File

@@ -1,7 +1,6 @@
package v1alpha1
import (
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/cli-utils/pkg/object"
)
@@ -15,82 +14,78 @@ type Drive interface {
SystemObjects() (objs []object.ObjMetadata)
}
//Driver
type Driver struct {
Kind string `json:"kind"`
Type string `json:"type"`
Version string `json:"version"`
}
type k3s struct {
dataDir string
etcDir string
}
//TODO: Don't hardcode this
func (k k3s) BinURL() string {
return "https://github.com/k3s-io/k3s/releases/download/v1.21.1%2Bk3s1/k3s"
}
func (k k3s) Images() ([]string, error) {
//TODO: Replace this with a query to images.txt on release page
return []string{
"docker.io/rancher/coredns-coredns:1.8.3",
"docker.io/rancher/klipper-helm:v0.5.0-build20210505",
"docker.io/rancher/klipper-lb:v0.2.0",
"docker.io/rancher/library-busybox:1.32.1",
"docker.io/rancher/library-traefik:2.4.8",
"docker.io/rancher/local-path-provisioner:v0.0.19",
"docker.io/rancher/metrics-server:v0.3.6",
"docker.io/rancher/pause:3.1",
}, nil
}
func (k k3s) Config() (*map[string]interface{}, error) {
// TODO: This should be typed
c := make(map[string]interface{})
c["write-kubeconfig-mode"] = "0644"
//TODO: Add uid or something to ensure this works for multi-node setups
c["node-name"] = "hauler"
return &c, nil
}
func (k k3s) SystemObjects() (objs []object.ObjMetadata) {
//TODO: Make sure this matches up with specified config disables
for _, dep := range []string{"coredns", "local-path-provisioner", "metrics-server"} {
objMeta, _ := object.CreateObjMetadata("kube-system", dep, schema.GroupKind{Kind: "Deployment", Group: "apps"})
objs = append(objs, objMeta)
}
return objs
}
func (k k3s) LibPath() string { return "/var/lib/rancher/k3s" }
func (k k3s) EtcPath() string { return "/etc/rancher/k3s" }
//TODO: Implement rke2 as a driver
type rke2 struct{}
func (r rke2) Images() ([]string, error) { return []string{}, nil }
func (r rke2) BinURL() string { return "" }
func (r rke2) LibPath() string { return "" }
func (r rke2) EtcPath() string { return "" }
func (r rke2) Config() (*map[string]interface{}, error) { return nil, nil }
func (r rke2) SystemObjects() (objs []object.ObjMetadata) { return objs }
//NewDriver will return the appropriate driver given a kind, defaults to k3s
func NewDriver(kind string) Drive {
var d Drive
switch kind {
case "rke2":
//TODO
d = rke2{}
default:
d = k3s{
dataDir: "/var/lib/rancher/k3s",
etcDir: "/etc/rancher/k3s",
}
}
return d
}
////TODO: Don't hardcode this
//func (k k3s) BinURL() string {
// return "https://github.com/k3s-io/k3s/releases/download/v1.21.1%2Bk3s1/k3s"
//}
//
//func (k k3s) PackageImages() ([]string, error) {
// //TODO: Replace this with a query to images.txt on release page
// return []string{
// "docker.io/rancher/coredns-coredns:1.8.3",
// "docker.io/rancher/klipper-helm:v0.5.0-build20210505",
// "docker.io/rancher/klipper-lb:v0.2.0",
// "docker.io/rancher/library-busybox:1.32.1",
// "docker.io/rancher/library-traefik:2.4.8",
// "docker.io/rancher/local-path-provisioner:v0.0.19",
// "docker.io/rancher/metrics-server:v0.3.6",
// "docker.io/rancher/pause:3.1",
// }, nil
//}
//
//func (k k3s) Config() (*map[string]interface{}, error) {
// // TODO: This should be typed
// c := make(map[string]interface{})
// c["write-kubeconfig-mode"] = "0644"
//
// //TODO: Add uid or something to ensure this works for multi-node setups
// c["node-name"] = "hauler"
//
// return &c, nil
//}
//
//func (k k3s) SystemObjects() (objs []object.ObjMetadata) {
// //TODO: Make sure this matches up with specified config disables
// for _, dep := range []string{"coredns", "local-path-provisioner", "metrics-server"} {
// objMeta, _ := object.CreateObjMetadata("kube-system", dep, schema.GroupKind{Kind: "Deployment", Group: "apps"})
// objs = append(objs, objMeta)
// }
// return objs
//}
//
//func (k k3s) LibPath() string { return "/var/lib/rancher/k3s" }
//func (k k3s) EtcPath() string { return "/etc/rancher/k3s" }
//
////TODO: Implement rke2 as a driver
//type rke2 struct{}
//
//func (r rke2) PackageImages() ([]string, error) { return []string{}, nil }
//func (r rke2) BinURL() string { return "" }
//func (r rke2) LibPath() string { return "" }
//func (r rke2) EtcPath() string { return "" }
//func (r rke2) Config() (*map[string]interface{}, error) { return nil, nil }
//func (r rke2) SystemObjects() (objs []object.ObjMetadata) { return objs }
//
////NewDriver will return the appropriate driver given a kind, defaults to k3s
//func NewDriver(kind string) Drive {
// var d Drive
// switch kind {
// case "rke2":
// //TODO
// d = rke2{}
//
// default:
// d = k3s{
// dataDir: "/var/lib/rancher/k3s",
// etcDir: "/etc/rancher/k3s",
// }
// }
//
// return d
//}

View File

@@ -5,9 +5,9 @@ import (
"context"
"fmt"
"github.com/google/go-containerregistry/pkg/v1/tarball"
"github.com/imdario/mergo"
"github.com/otiai10/copy"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
"github.com/rancherfederal/hauler/pkg/driver"
"github.com/rancherfederal/hauler/pkg/fs"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/sirupsen/logrus"
@@ -15,16 +15,14 @@ import (
"io"
"k8s.io/cli-runtime/pkg/genericclioptions"
"os"
"os/exec"
"path/filepath"
"sigs.k8s.io/yaml"
)
type Booter interface {
Init() error
PreBoot(context.Context) error
Boot(context.Context, v1alpha1.Drive) error
PostBoot(context.Context, v1alpha1.Drive) error
Boot(context.Context, driver.Driver) error
PostBoot(context.Context, driver.Driver) error
}
type booter struct {
@@ -47,68 +45,49 @@ func NewBooter(pkgPath string) (*booter, error) {
}, nil
}
func (b booter) Init() error {
d := v1alpha1.NewDriver(b.Package.Spec.Driver.Kind)
func (b booter) PreBoot(ctx context.Context, d driver.Driver, logger log.Logger) error {
l := logger.WithFields(logrus.Fields{
"phase": "preboot",
})
//TODO: Feel like there's a better way to do this
if err := b.moveBin(); err != nil {
return err
}
if err := b.moveImages(d); err != nil {
return err
}
if err := b.moveBundles(d); err != nil {
return err
}
if err := b.moveCharts(d); err != nil {
return err
}
return nil
}
func (b booter) PreBoot(ctx context.Context, d v1alpha1.Drive, logger log.Logger) error {
l := logger.WithFields(logrus.Fields{
"phase": "preboot",
})
l.Infof("Creating driver configuration")
if err := b.writeConfig(d); err != nil {
if err := d.WriteConfig(); err != nil {
return err
}
return nil
}
func (b booter) Boot(ctx context.Context, d v1alpha1.Drive, logger log.Logger) error {
func (b booter) Boot(ctx context.Context, d driver.Driver, logger log.Logger) error {
l := logger.WithFields(logrus.Fields{
"phase": "boot",
})
//TODO: Generic
cmd := exec.Command("/bin/sh", "/opt/hauler/bin/k3s-init.sh")
cmd.Env = append(os.Environ(), []string{
"INSTALL_K3S_SKIP_DOWNLOAD=true",
"INSTALL_K3S_SELINUX_WARN=true",
"INSTALL_K3S_SKIP_SELINUX_RPM=true",
"INSTALL_K3S_BIN_DIR=/opt/hauler/bin",
//TODO: Provide a real dryrun option
//"INSTALL_K3S_SKIP_START=true",
}...)
var stdoutBuf, stderrBuf bytes.Buffer
cmd.Stdout = io.MultiWriter(os.Stdout, &stdoutBuf)
cmd.Stderr = io.MultiWriter(os.Stderr, &stderrBuf)
out := io.MultiWriter(os.Stdout, &stdoutBuf, &stderrBuf)
err := cmd.Run()
err := d.Start(out)
if err != nil {
return err
}
l.Infof("Driver successfully started!")
l.Infof("Waiting for driver core components to provision...")
waitErr := waitForDriver(ctx, d)
if waitErr != nil {
@@ -118,13 +97,13 @@ func (b booter) Boot(ctx context.Context, d v1alpha1.Drive, logger log.Logger) e
return nil
}
func (b booter) PostBoot(ctx context.Context, d v1alpha1.Drive, logger log.Logger) error {
func (b booter) PostBoot(ctx context.Context, d driver.Driver, logger log.Logger) error {
l := logger.WithFields(logrus.Fields{
"phase": "postboot",
})
cf := genericclioptions.NewConfigFlags(true)
cf.KubeConfig = stringptr(fmt.Sprintf("%s/k3s.yaml", d.EtcPath()))
cf.KubeConfig = stringptr(d.KubeConfigPath())
fleetCrdChartPath := b.fs.Chart().Path(fmt.Sprintf("fleet-crd-%s.tgz", b.Package.Spec.Fleet.Version))
fleetCrdChart, err := loader.Load(fleetCrdChartPath)
@@ -167,9 +146,9 @@ func (b booter) moveBin() error {
return copy.Copy(b.fs.Bin().Path(), path)
}
func (b booter) moveImages(d v1alpha1.Drive) error {
func (b booter) moveImages(d driver.Driver) error {
//NOTE: archives are not recursively searched, this _must_ be at the images dir
path := filepath.Join(d.LibPath(), "agent/images")
path := d.DataPath("agent/images")
if err := os.MkdirAll(path, 0700); err != nil {
return err
}
@@ -182,49 +161,18 @@ func (b booter) moveImages(d v1alpha1.Drive) error {
return tarball.MultiRefWriteToFile(filepath.Join(path, "hauler.tar"), refs)
}
func (b booter) moveBundles(d v1alpha1.Drive) error {
path := filepath.Join(d.LibPath(), "server/manifests/hauler")
if err := os.MkdirAll(d.LibPath(), 0700); err != nil {
return err
}
return copy.Copy(b.fs.Bundle().Path(), path)
}
func (b booter) moveCharts(d v1alpha1.Drive) error {
path := filepath.Join(d.LibPath(), "server/static/charts/hauler")
func (b booter) moveBundles(d driver.Driver) error {
path := d.DataPath("server/manifests/hauler")
if err := os.MkdirAll(path, 0700); err != nil {
return err
}
return copy.Copy(b.fs.Bundle().Path(), path)
}
func (b booter) moveCharts(d driver.Driver) error {
path := d.DataPath("server/static/charts/hauler")
if err := os.MkdirAll(path, 0700); err != nil {
return err
}
return copy.Copy(b.fs.Chart().Path(), path)
}
func (b booter) writeConfig(d v1alpha1.Drive) error {
if err := os.MkdirAll(d.EtcPath(), os.ModePerm); err != nil {
return err
}
c, err := d.Config()
if err != nil {
return err
}
var uc map[string]interface{}
path := filepath.Join(d.EtcPath(), "config.yaml")
if data, err := os.ReadFile(path); err != nil {
err := yaml.Unmarshal(data, &uc)
if err != nil {
return err
}
}
//Merge with user defined configs taking precedence
if err := mergo.Merge(c, uc); err != nil {
return err
}
data, err := yaml.Marshal(c)
return os.WriteFile(path, data, 0644)
}

View File

@@ -3,7 +3,7 @@ package bootstrap
import (
"context"
"errors"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
"github.com/rancherfederal/hauler/pkg/driver"
"github.com/rancherfederal/hauler/pkg/kube"
log "github.com/sirupsen/logrus"
"helm.sh/helm/v3/pkg/action"
@@ -11,18 +11,16 @@ import (
"helm.sh/helm/v3/pkg/release"
"k8s.io/cli-runtime/pkg/genericclioptions"
"os"
"path/filepath"
"time"
)
func waitForDriver(ctx context.Context, d v1alpha1.Drive) error {
func waitForDriver(ctx context.Context, d driver.Driver) error {
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
//TODO: This is a janky way of waiting for file to exist
path := filepath.Join(d.EtcPath(), "k3s.yaml")
for {
_, err := os.Stat(path)
_, err := os.Stat(d.KubeConfigPath())
if err == nil {
break
}
@@ -56,10 +54,12 @@ func installChart(cf *genericclioptions.ConfigFlags, chart *chart.Chart, release
client := action.NewInstall(actionConfig)
client.ReleaseName = releaseName
client.Namespace, cf.Namespace = namespace, stringptr(namespace) // TODO: Not sure why this needs to be set twice
client.CreateNamespace = true
client.Wait = true
//TODO: Do this better
client.Namespace, cf.Namespace = namespace, stringptr(namespace)
return client.Run(chart, vals)
}

49
pkg/driver/driver.go Normal file
View File

@@ -0,0 +1,49 @@
package driver
import (
"context"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
"io"
"sigs.k8s.io/cli-utils/pkg/object"
)
type Driver interface {
Name() string
//TODO: Really want this to just return a usable client
KubeConfigPath() string
Images(ctx context.Context) (map[name.Reference]v1.Image, error)
Binary() (io.ReadCloser, error)
SystemObjects() []object.ObjMetadata
Start(io.Writer) error
DataPath(...string) string
WriteConfig() error
}
//NewDriver will return a new concrete Driver type given a kind
func NewDriver(driver v1alpha1.Driver) (d Driver) {
switch driver.Type {
case "rke2":
// TODO
default:
d = K3s{
Version: driver.Version,
Config: K3sConfig{
DataDir: "/var/lib/rancher/k3s",
KubeConfig: "/etc/rancher/k3s/k3s.yaml",
KubeConfigMode: "0644",
Disable: nil,
},
}
}
return
}

View File

@@ -0,0 +1,872 @@
#!/bin/sh
set -e
set -o noglob
# Usage:
# curl ... | ENV_VAR=... sh -
# or
# ENV_VAR=... ./install.sh
#
# Example:
# Installing a server without traefik:
# curl ... | INSTALL_K3S_EXEC="--disable=traefik" sh -
# Installing an agent to point at a server:
# curl ... | K3S_TOKEN=xxx K3S_URL=https://server-url:6443 sh -
#
# Environment variables:
# - K3S_*
# Environment variables which begin with K3S_ will be preserved for the
# systemd service to use. Setting K3S_URL without explicitly setting
# a systemd exec command will default the command to "agent", and we
# enforce that K3S_TOKEN or K3S_CLUSTER_SECRET is also set.
#
# - INSTALL_K3S_SKIP_DOWNLOAD
# If set to true will not download k3s hash or binary.
#
# - INSTALL_K3S_FORCE_RESTART
# If set to true will always restart the K3s service
#
# - INSTALL_K3S_SYMLINK
# If set to 'skip' will not create symlinks, 'force' will overwrite,
# default will symlink if command does not exist in path.
#
# - INSTALL_K3S_SKIP_ENABLE
# If set to true will not enable or start k3s service.
#
# - INSTALL_K3S_SKIP_START
# If set to true will not start k3s service.
#
# - INSTALL_K3S_VERSION
# Version of k3s to download from github. Will attempt to download from the
# stable channel if not specified.
#
# - INSTALL_K3S_COMMIT
# Commit of k3s to download from temporary cloud storage.
# * (for developer & QA use)
#
# - INSTALL_K3S_BIN_DIR
# Directory to install k3s binary, links, and uninstall script to, or use
# /usr/local/bin as the default
#
# - INSTALL_K3S_BIN_DIR_READ_ONLY
# If set to true will not write files to INSTALL_K3S_BIN_DIR, forces
# setting INSTALL_K3S_SKIP_DOWNLOAD=true
#
# - INSTALL_K3S_SYSTEMD_DIR
# Directory to install systemd service and environment files to, or use
# /etc/systemd/system as the default
#
# - INSTALL_K3S_EXEC or script arguments
# Command with flags to use for launching k3s in the systemd service, if
# the command is not specified will default to "agent" if K3S_URL is set
# or "server" if not. The final systemd command resolves to a combination
# of EXEC and script args ($@).
#
# The following commands result in the same behavior:
# curl ... | INSTALL_K3S_EXEC="--disable=traefik" sh -s -
# curl ... | INSTALL_K3S_EXEC="server --disable=traefik" sh -s -
# curl ... | INSTALL_K3S_EXEC="server" sh -s - --disable=traefik
# curl ... | sh -s - server --disable=traefik
# curl ... | sh -s - --disable=traefik
#
# - INSTALL_K3S_NAME
# Name of systemd service to create, will default from the k3s exec command
# if not specified. If specified the name will be prefixed with 'k3s-'.
#
# - INSTALL_K3S_TYPE
# Type of systemd service to create, will default from the k3s exec command
# if not specified.
#
# - INSTALL_K3S_SELINUX_WARN
# If set to true will continue if k3s-selinux policy is not found.
#
# - INSTALL_K3S_SKIP_SELINUX_RPM
# If set to true will skip automatic installation of the k3s RPM.
#
# - INSTALL_K3S_CHANNEL_URL
# Channel URL for fetching k3s download URL.
# Defaults to 'https://update.k3s.io/v1-release/channels'.
#
# - INSTALL_K3S_CHANNEL
# Channel to use for fetching k3s download URL.
# Defaults to 'stable'.
GITHUB_URL=https://github.com/k3s-io/k3s/releases
STORAGE_URL=https://storage.googleapis.com/k3s-ci-builds
DOWNLOADER=
# --- helper functions for logs ---
info()
{
echo '[INFO] ' "$@"
}
warn()
{
echo '[WARN] ' "$@" >&2
}
fatal()
{
echo '[ERROR] ' "$@" >&2
exit 1
}
# --- fatal if no systemd or openrc ---
verify_system() {
if [ -x /sbin/openrc-run ]; then
HAS_OPENRC=true
return
fi
if [ -d /run/systemd ]; then
HAS_SYSTEMD=true
return
fi
fatal 'Can not find systemd or openrc to use as a process supervisor for k3s'
}
# --- add quotes to command arguments ---
quote() {
for arg in "$@"; do
printf '%s\n' "$arg" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
done
}
# --- add indentation and trailing slash to quoted args ---
quote_indent() {
printf ' \\\n'
for arg in "$@"; do
printf '\t%s \\\n' "$(quote "$arg")"
done
}
# --- escape most punctuation characters, except quotes, forward slash, and space ---
escape() {
printf '%s' "$@" | sed -e 's/\([][!#$%&()*;<=>?\_`{|}]\)/\\\1/g;'
}
# --- escape double quotes ---
escape_dq() {
printf '%s' "$@" | sed -e 's/"/\\"/g'
}
# --- ensures $K3S_URL is empty or begins with https://, exiting fatally otherwise ---
verify_k3s_url() {
case "${K3S_URL}" in
"")
;;
https://*)
;;
*)
fatal "Only https:// URLs are supported for K3S_URL (have ${K3S_URL})"
;;
esac
}
# --- define needed environment variables ---
setup_env() {
# --- use command args if passed or create default ---
case "$1" in
# --- if we only have flags discover if command should be server or agent ---
(-*|"")
if [ -z "${K3S_URL}" ]; then
CMD_K3S=server
else
if [ -z "${K3S_TOKEN}" ] && [ -z "${K3S_TOKEN_FILE}" ] && [ -z "${K3S_CLUSTER_SECRET}" ]; then
fatal "Defaulted k3s exec command to 'agent' because K3S_URL is defined, but K3S_TOKEN, K3S_TOKEN_FILE or K3S_CLUSTER_SECRET is not defined."
fi
CMD_K3S=agent
fi
;;
# --- command is provided ---
(*)
CMD_K3S=$1
shift
;;
esac
verify_k3s_url
CMD_K3S_EXEC="${CMD_K3S}$(quote_indent "$@")"
# --- use systemd name if defined or create default ---
if [ -n "${INSTALL_K3S_NAME}" ]; then
SYSTEM_NAME=k3s-${INSTALL_K3S_NAME}
else
if [ "${CMD_K3S}" = server ]; then
SYSTEM_NAME=k3s
else
SYSTEM_NAME=k3s-${CMD_K3S}
fi
fi
# --- check for invalid characters in system name ---
valid_chars=$(printf '%s' "${SYSTEM_NAME}" | sed -e 's/[][!#$%&()*;<=>?\_`{|}/[:space:]]/^/g;' )
if [ "${SYSTEM_NAME}" != "${valid_chars}" ]; then
invalid_chars=$(printf '%s' "${valid_chars}" | sed -e 's/[^^]/ /g')
fatal "Invalid characters for system name:
${SYSTEM_NAME}
${invalid_chars}"
fi
# --- use sudo if we are not already root ---
SUDO=sudo
if [ $(id -u) -eq 0 ]; then
SUDO=
fi
# --- use systemd type if defined or create default ---
if [ -n "${INSTALL_K3S_TYPE}" ]; then
SYSTEMD_TYPE=${INSTALL_K3S_TYPE}
else
if [ "${CMD_K3S}" = server ]; then
SYSTEMD_TYPE=notify
else
SYSTEMD_TYPE=exec
fi
fi
# --- use binary install directory if defined or create default ---
if [ -n "${INSTALL_K3S_BIN_DIR}" ]; then
BIN_DIR=${INSTALL_K3S_BIN_DIR}
else
# --- use /usr/local/bin if root can write to it, otherwise use /opt/bin if it exists
BIN_DIR=/usr/local/bin
if ! $SUDO sh -c "touch ${BIN_DIR}/k3s-ro-test && rm -rf ${BIN_DIR}/k3s-ro-test"; then
if [ -d /opt/bin ]; then
BIN_DIR=/opt/bin
fi
fi
fi
# --- use systemd directory if defined or create default ---
if [ -n "${INSTALL_K3S_SYSTEMD_DIR}" ]; then
SYSTEMD_DIR="${INSTALL_K3S_SYSTEMD_DIR}"
else
SYSTEMD_DIR=/etc/systemd/system
fi
# --- set related files from system name ---
SERVICE_K3S=${SYSTEM_NAME}.service
UNINSTALL_K3S_SH=${UNINSTALL_K3S_SH:-${BIN_DIR}/${SYSTEM_NAME}-uninstall.sh}
KILLALL_K3S_SH=${KILLALL_K3S_SH:-${BIN_DIR}/k3s-killall.sh}
# --- use service or environment location depending on systemd/openrc ---
if [ "${HAS_SYSTEMD}" = true ]; then
FILE_K3S_SERVICE=${SYSTEMD_DIR}/${SERVICE_K3S}
FILE_K3S_ENV=${SYSTEMD_DIR}/${SERVICE_K3S}.env
elif [ "${HAS_OPENRC}" = true ]; then
$SUDO mkdir -p /etc/rancher/k3s
FILE_K3S_SERVICE=/etc/init.d/${SYSTEM_NAME}
FILE_K3S_ENV=/etc/rancher/k3s/${SYSTEM_NAME}.env
fi
# --- get hash of config & exec for currently installed k3s ---
PRE_INSTALL_HASHES=$(get_installed_hashes)
# --- if bin directory is read only skip download ---
if [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ]; then
INSTALL_K3S_SKIP_DOWNLOAD=true
fi
# --- setup channel values
INSTALL_K3S_CHANNEL_URL=${INSTALL_K3S_CHANNEL_URL:-'https://update.k3s.io/v1-release/channels'}
INSTALL_K3S_CHANNEL=${INSTALL_K3S_CHANNEL:-'stable'}
}
# --- check if skip download environment variable set ---
can_skip_download() {
if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != true ]; then
return 1
fi
}
# --- verify an executable k3s binary is installed ---
verify_k3s_is_executable() {
if [ ! -x ${BIN_DIR}/k3s ]; then
fatal "Executable k3s binary not found at ${BIN_DIR}/k3s"
fi
}
# --- set arch and suffix, fatal if architecture not supported ---
setup_verify_arch() {
if [ -z "$ARCH" ]; then
ARCH=$(uname -m)
fi
case $ARCH in
amd64)
ARCH=amd64
SUFFIX=
;;
x86_64)
ARCH=amd64
SUFFIX=
;;
arm64)
ARCH=arm64
SUFFIX=-${ARCH}
;;
aarch64)
ARCH=arm64
SUFFIX=-${ARCH}
;;
arm*)
ARCH=arm
SUFFIX=-${ARCH}hf
;;
*)
fatal "Unsupported architecture $ARCH"
esac
}
# --- verify existence of network downloader executable ---
verify_downloader() {
# Return failure if it doesn't exist or is no executable
[ -x "$(command -v $1)" ] || return 1
# Set verified executable as our downloader program and return success
DOWNLOADER=$1
return 0
}
# --- create temporary directory and cleanup when done ---
setup_tmp() {
TMP_DIR=$(mktemp -d -t k3s-install.XXXXXXXXXX)
TMP_HASH=${TMP_DIR}/k3s.hash
TMP_BIN=${TMP_DIR}/k3s.bin
cleanup() {
code=$?
set +e
trap - EXIT
rm -rf ${TMP_DIR}
exit $code
}
trap cleanup INT EXIT
}
# --- use desired k3s version if defined or find version from channel ---
get_release_version() {
if [ -n "${INSTALL_K3S_COMMIT}" ]; then
VERSION_K3S="commit ${INSTALL_K3S_COMMIT}"
elif [ -n "${INSTALL_K3S_VERSION}" ]; then
VERSION_K3S=${INSTALL_K3S_VERSION}
else
info "Finding release for channel ${INSTALL_K3S_CHANNEL}"
version_url="${INSTALL_K3S_CHANNEL_URL}/${INSTALL_K3S_CHANNEL}"
case $DOWNLOADER in
curl)
VERSION_K3S=$(curl -w '%{url_effective}' -L -s -S ${version_url} -o /dev/null | sed -e 's|.*/||')
;;
wget)
VERSION_K3S=$(wget -SqO /dev/null ${version_url} 2>&1 | grep -i Location | sed -e 's|.*/||')
;;
*)
fatal "Incorrect downloader executable '$DOWNLOADER'"
;;
esac
fi
info "Using ${VERSION_K3S} as release"
}
# --- download from github url ---
download() {
[ $# -eq 2 ] || fatal 'download needs exactly 2 arguments'
case $DOWNLOADER in
curl)
curl -o $1 -sfL $2
;;
wget)
wget -qO $1 $2
;;
*)
fatal "Incorrect executable '$DOWNLOADER'"
;;
esac
# Abort if download command failed
[ $? -eq 0 ] || fatal 'Download failed'
}
# --- download hash from github url ---
download_hash() {
if [ -n "${INSTALL_K3S_COMMIT}" ]; then
HASH_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT}.sha256sum
else
HASH_URL=${GITHUB_URL}/download/${VERSION_K3S}/sha256sum-${ARCH}.txt
fi
info "Downloading hash ${HASH_URL}"
download ${TMP_HASH} ${HASH_URL}
HASH_EXPECTED=$(grep " k3s${SUFFIX}$" ${TMP_HASH})
HASH_EXPECTED=${HASH_EXPECTED%%[[:blank:]]*}
}
# --- check hash against installed version ---
installed_hash_matches() {
if [ -x ${BIN_DIR}/k3s ]; then
HASH_INSTALLED=$(sha256sum ${BIN_DIR}/k3s)
HASH_INSTALLED=${HASH_INSTALLED%%[[:blank:]]*}
if [ "${HASH_EXPECTED}" = "${HASH_INSTALLED}" ]; then
return
fi
fi
return 1
}
# --- download binary from github url ---
download_binary() {
if [ -n "${INSTALL_K3S_COMMIT}" ]; then
BIN_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT}
else
BIN_URL=${GITHUB_URL}/download/${VERSION_K3S}/k3s${SUFFIX}
fi
info "Downloading binary ${BIN_URL}"
download ${TMP_BIN} ${BIN_URL}
}
# --- verify downloaded binary hash ---
verify_binary() {
info "Verifying binary download"
HASH_BIN=$(sha256sum ${TMP_BIN})
HASH_BIN=${HASH_BIN%%[[:blank:]]*}
if [ "${HASH_EXPECTED}" != "${HASH_BIN}" ]; then
fatal "Download sha256 does not match ${HASH_EXPECTED}, got ${HASH_BIN}"
fi
}
# --- setup permissions and move binary to system directory ---
setup_binary() {
chmod 755 ${TMP_BIN}
info "Installing k3s to ${BIN_DIR}/k3s"
$SUDO chown root:root ${TMP_BIN}
$SUDO mv -f ${TMP_BIN} ${BIN_DIR}/k3s
}
# --- setup selinux policy ---
setup_selinux() {
case ${INSTALL_K3S_CHANNEL} in
*testing)
rpm_channel=testing
;;
*latest)
rpm_channel=latest
;;
*)
rpm_channel=stable
;;
esac
rpm_site="rpm.rancher.io"
if [ "${rpm_channel}" = "testing" ]; then
rpm_site="rpm-testing.rancher.io"
fi
policy_hint="please install:
yum install -y container-selinux selinux-policy-base
yum install -y https://${rpm_site}/k3s/${rpm_channel}/common/centos/7/noarch/k3s-selinux-0.2-1.el7_8.noarch.rpm
"
policy_error=fatal
if [ "$INSTALL_K3S_SELINUX_WARN" = true ] || grep -q 'ID=flatcar' /etc/os-release; then
policy_error=warn
fi
if [ "$INSTALL_K3S_SKIP_SELINUX_RPM" = true ] || can_skip_download; then
info "Skipping installation of SELinux RPM"
else
install_selinux_rpm ${rpm_site} ${rpm_channel}
fi
if ! $SUDO chcon -u system_u -r object_r -t container_runtime_exec_t ${BIN_DIR}/k3s >/dev/null 2>&1; then
if $SUDO grep '^\s*SELINUX=enforcing' /etc/selinux/config >/dev/null 2>&1; then
$policy_error "Failed to apply container_runtime_exec_t to ${BIN_DIR}/k3s, ${policy_hint}"
fi
else
if [ ! -f /usr/share/selinux/packages/k3s.pp ]; then
$policy_error "Failed to find the k3s-selinux policy, ${policy_hint}"
fi
fi
}
# --- if on an el7/el8 system, install k3s-selinux
install_selinux_rpm() {
if [ -r /etc/redhat-release ] || [ -r /etc/centos-release ] || [ -r /etc/oracle-release ]; then
dist_version="$(. /etc/os-release && echo "$VERSION_ID")"
maj_ver=$(echo "$dist_version" | sed -E -e "s/^([0-9]+)\.?[0-9]*$/\1/")
set +o noglob
$SUDO rm -f /etc/yum.repos.d/rancher-k3s-common*.repo
set -o noglob
if [ -r /etc/redhat-release ]; then
case ${maj_ver} in
7)
$SUDO yum -y install yum-utils
$SUDO yum-config-manager --enable rhel-7-server-extras-rpms
;;
8)
:
;;
*)
return
;;
esac
fi
$SUDO tee /etc/yum.repos.d/rancher-k3s-common.repo >/dev/null << EOF
[rancher-k3s-common-${2}]
name=Rancher K3s Common (${2})
baseurl=https://${1}/k3s/${2}/common/centos/${maj_ver}/noarch
enabled=1
gpgcheck=1
gpgkey=https://${1}/public.key
EOF
$SUDO yum -y install "k3s-selinux"
fi
return
}
# --- download and verify k3s ---
download_and_verify() {
if can_skip_download; then
info 'Skipping k3s download and verify'
verify_k3s_is_executable
return
fi
setup_verify_arch
verify_downloader curl || verify_downloader wget || fatal 'Can not find curl or wget for downloading files'
setup_tmp
get_release_version
download_hash
if installed_hash_matches; then
info 'Skipping binary downloaded, installed k3s matches hash'
return
fi
download_binary
verify_binary
setup_binary
}
# --- add additional utility links ---
create_symlinks() {
[ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return
[ "${INSTALL_K3S_SYMLINK}" = skip ] && return
for cmd in kubectl crictl ctr; do
if [ ! -e ${BIN_DIR}/${cmd} ] || [ "${INSTALL_K3S_SYMLINK}" = force ]; then
which_cmd=$(command -v ${cmd} 2>/dev/null || true)
if [ -z "${which_cmd}" ] || [ "${INSTALL_K3S_SYMLINK}" = force ]; then
info "Creating ${BIN_DIR}/${cmd} symlink to k3s"
$SUDO ln -sf k3s ${BIN_DIR}/${cmd}
else
info "Skipping ${BIN_DIR}/${cmd} symlink to k3s, command exists in PATH at ${which_cmd}"
fi
else
info "Skipping ${BIN_DIR}/${cmd} symlink to k3s, already exists"
fi
done
}
# --- create killall script ---
create_killall() {
[ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return
info "Creating killall script ${KILLALL_K3S_SH}"
$SUDO tee ${KILLALL_K3S_SH} >/dev/null << \EOF
#!/bin/sh
[ $(id -u) -eq 0 ] || exec sudo $0 $@
for bin in /var/lib/rancher/k3s/data/**/bin/; do
[ -d $bin ] && export PATH=$PATH:$bin:$bin/aux
done
set -x
for service in /etc/systemd/system/k3s*.service; do
[ -s $service ] && systemctl stop $(basename $service)
done
for service in /etc/init.d/k3s*; do
[ -x $service ] && $service stop
done
pschildren() {
ps -e -o ppid= -o pid= | \
sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \
grep -w "^$1" | \
cut -f2
}
pstree() {
for pid in $@; do
echo $pid
for child in $(pschildren $pid); do
pstree $child
done
done
}
killtree() {
kill -9 $(
{ set +x; } 2>/dev/null;
pstree $@;
set -x;
) 2>/dev/null
}
getshims() {
ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1
}
killtree $({ set +x; } 2>/dev/null; getshims; set -x)
do_unmount_and_remove() {
awk -v path="$1" '$2 ~ ("^" path) { print $2 }' /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount "$0" && rm -rf "$0"'
}
do_unmount_and_remove '/run/k3s'
do_unmount_and_remove '/var/lib/rancher/k3s'
do_unmount_and_remove '/var/lib/kubelet/pods'
do_unmount_and_remove '/var/lib/kubelet/plugins'
do_unmount_and_remove '/run/netns/cni-'
# Remove CNI namespaces
ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete
# Delete network interface(s) that match 'master cni0'
ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do
iface=${iface%%@*}
[ -z "$iface" ] || ip link delete $iface
done
ip link delete cni0
ip link delete flannel.1
rm -rf /var/lib/cni/
iptables-save | grep -v KUBE- | grep -v CNI- | iptables-restore
EOF
$SUDO chmod 755 ${KILLALL_K3S_SH}
$SUDO chown root:root ${KILLALL_K3S_SH}
}
# --- create uninstall script ---
create_uninstall() {
[ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return
info "Creating uninstall script ${UNINSTALL_K3S_SH}"
$SUDO tee ${UNINSTALL_K3S_SH} >/dev/null << EOF
#!/bin/sh
set -x
[ \$(id -u) -eq 0 ] || exec sudo \$0 \$@
${KILLALL_K3S_SH}
if command -v systemctl; then
systemctl disable ${SYSTEM_NAME}
systemctl reset-failed ${SYSTEM_NAME}
systemctl daemon-reload
fi
if command -v rc-update; then
rc-update delete ${SYSTEM_NAME} default
fi
rm -f ${FILE_K3S_SERVICE}
rm -f ${FILE_K3S_ENV}
remove_uninstall() {
rm -f ${UNINSTALL_K3S_SH}
}
trap remove_uninstall EXIT
if (ls ${SYSTEMD_DIR}/k3s*.service || ls /etc/init.d/k3s*) >/dev/null 2>&1; then
set +x; echo 'Additional k3s services installed, skipping uninstall of k3s'; set -x
exit
fi
for cmd in kubectl crictl ctr; do
if [ -L ${BIN_DIR}/\$cmd ]; then
rm -f ${BIN_DIR}/\$cmd
fi
done
rm -rf /etc/rancher/k3s
rm -rf /run/k3s
rm -rf /run/flannel
rm -rf /var/lib/rancher/k3s
rm -rf /var/lib/kubelet
rm -f ${BIN_DIR}/k3s
rm -f ${KILLALL_K3S_SH}
if type yum >/dev/null 2>&1; then
yum remove -y k3s-selinux
rm -f /etc/yum.repos.d/rancher-k3s-common*.repo
fi
EOF
$SUDO chmod 755 ${UNINSTALL_K3S_SH}
$SUDO chown root:root ${UNINSTALL_K3S_SH}
}
# --- disable current service if loaded --
systemd_disable() {
$SUDO systemctl disable ${SYSTEM_NAME} >/dev/null 2>&1 || true
$SUDO rm -f /etc/systemd/system/${SERVICE_K3S} || true
$SUDO rm -f /etc/systemd/system/${SERVICE_K3S}.env || true
}
# --- capture current env and create file containing k3s_ variables ---
create_env_file() {
info "env: Creating environment file ${FILE_K3S_ENV}"
$SUDO touch ${FILE_K3S_ENV}
$SUDO chmod 0600 ${FILE_K3S_ENV}
env | grep '^K3S_' | $SUDO tee ${FILE_K3S_ENV} >/dev/null
env | grep -Ei '^(NO|HTTP|HTTPS)_PROXY' | $SUDO tee -a ${FILE_K3S_ENV} >/dev/null
}
# --- write systemd service file ---
create_systemd_service_file() {
info "systemd: Creating service file ${FILE_K3S_SERVICE}"
$SUDO tee ${FILE_K3S_SERVICE} >/dev/null << EOF
[Unit]
Description=Lightweight Kubernetes
Documentation=https://k3s.io
Wants=network-online.target
After=network-online.target
[Install]
WantedBy=multi-user.target
[Service]
Type=${SYSTEMD_TYPE}
EnvironmentFile=-/etc/default/%N
EnvironmentFile=-/etc/sysconfig/%N
EnvironmentFile=-${FILE_K3S_ENV}
KillMode=process
Delegate=yes
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=1048576
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity
TimeoutStartSec=0
Restart=always
RestartSec=5s
ExecStartPre=-/sbin/modprobe br_netfilter
ExecStartPre=-/sbin/modprobe overlay
ExecStart=${BIN_DIR}/k3s \\
${CMD_K3S_EXEC}
EOF
}
# --- write openrc service file ---
create_openrc_service_file() {
LOG_FILE=/var/log/${SYSTEM_NAME}.log
info "openrc: Creating service file ${FILE_K3S_SERVICE}"
$SUDO tee ${FILE_K3S_SERVICE} >/dev/null << EOF
#!/sbin/openrc-run
depend() {
after network-online
want cgroups
}
start_pre() {
rm -f /tmp/k3s.*
}
supervisor=supervise-daemon
name=${SYSTEM_NAME}
command="${BIN_DIR}/k3s"
command_args="$(escape_dq "${CMD_K3S_EXEC}")
>>${LOG_FILE} 2>&1"
output_log=${LOG_FILE}
error_log=${LOG_FILE}
pidfile="/var/run/${SYSTEM_NAME}.pid"
respawn_delay=5
respawn_max=0
set -o allexport
if [ -f /etc/environment ]; then source /etc/environment; fi
if [ -f ${FILE_K3S_ENV} ]; then source ${FILE_K3S_ENV}; fi
set +o allexport
EOF
$SUDO chmod 0755 ${FILE_K3S_SERVICE}
$SUDO tee /etc/logrotate.d/${SYSTEM_NAME} >/dev/null << EOF
${LOG_FILE} {
missingok
notifempty
copytruncate
}
EOF
}
# --- write systemd or openrc service file ---
create_service_file() {
[ "${HAS_SYSTEMD}" = true ] && create_systemd_service_file
[ "${HAS_OPENRC}" = true ] && create_openrc_service_file
return 0
}
# --- get hashes of the current k3s bin and service files
get_installed_hashes() {
$SUDO sha256sum ${BIN_DIR}/k3s ${FILE_K3S_SERVICE} ${FILE_K3S_ENV} 2>&1 || true
}
# --- enable and start systemd service ---
systemd_enable() {
info "systemd: Enabling ${SYSTEM_NAME} unit"
$SUDO systemctl enable ${FILE_K3S_SERVICE} >/dev/null
$SUDO systemctl daemon-reload >/dev/null
}
systemd_start() {
info "systemd: Starting ${SYSTEM_NAME}"
$SUDO systemctl restart ${SYSTEM_NAME}
}
# --- enable and start openrc service ---
openrc_enable() {
info "openrc: Enabling ${SYSTEM_NAME} service for default runlevel"
$SUDO rc-update add ${SYSTEM_NAME} default >/dev/null
}
openrc_start() {
info "openrc: Starting ${SYSTEM_NAME}"
$SUDO ${FILE_K3S_SERVICE} restart
}
# --- startup systemd or openrc service ---
service_enable_and_start() {
[ "${INSTALL_K3S_SKIP_ENABLE}" = true ] && return
[ "${HAS_SYSTEMD}" = true ] && systemd_enable
[ "${HAS_OPENRC}" = true ] && openrc_enable
[ "${INSTALL_K3S_SKIP_START}" = true ] && return
POST_INSTALL_HASHES=$(get_installed_hashes)
if [ "${PRE_INSTALL_HASHES}" = "${POST_INSTALL_HASHES}" ] && [ "${INSTALL_K3S_FORCE_RESTART}" != true ]; then
info 'No change detected so skipping service start'
return
fi
[ "${HAS_SYSTEMD}" = true ] && systemd_start
[ "${HAS_OPENRC}" = true ] && openrc_start
return 0
}
# --- re-evaluate args to include env command ---
eval set -- $(escape "${INSTALL_K3S_EXEC}") $(quote "$@")
# --- run the install process --
{
verify_system
setup_env "$@"
download_and_verify
setup_selinux
create_symlinks
create_killall
create_uninstall
systemd_disable
create_env_file
create_service_file
service_enable_and_start
}

View File

@@ -0,0 +1,507 @@
#!/bin/sh
set -e
if [ "${DEBUG}" = 1 ]; then
set -x
fi
# Usage:
# curl ... | ENV_VAR=... sh -
# or
# ENV_VAR=... ./install.sh
#
# Environment variables:
#
# - INSTALL_RKE2_CHANNEL
# Channel to use for fetching rke2 download URL.
# Defaults to 'latest'.
#
# - INSTALL_RKE2_METHOD
# The installation method to use.
# Default is on RPM-based systems is "rpm", all else "tar".
#
# - INSTALL_RKE2_TYPE
# Type of rke2 service. Can be either "server" or "agent".
# Default is "server".
#
# - INSTALL_RKE2_EXEC
# This is an alias for INSTALL_RKE2_TYPE, included for compatibility with K3s.
# If both are set, INSTALL_RKE2_TYPE is preferred.
#
# - INSTALL_RKE2_VERSION
# Version of rke2 to download from github.
#
# - INSTALL_RKE2_RPM_RELEASE_VERSION
# Version of the rke2 RPM release to install.
# Format would be like "1.el7" or "2.el8"
#
# - INSTALL_RKE2_TAR_PREFIX
# Installation prefix when using the tar installation method.
# Default is /usr/local, unless /usr/local is read-only or has a dedicated mount point,
# in which case /opt/rke2 is used instead.
#
# - INSTALL_RKE2_COMMIT
# Commit of RKE2 to download from temporary cloud storage.
# If set, this forces INSTALL_RKE2_METHOD=tar.
# * (for developer & QA use)
#
# - INSTALL_RKE2_AGENT_IMAGES_DIR
# Installation path for airgap images when installing from CI commit
# Default is /var/lib/rancher/rke2/agent/images
#
# - INSTALL_RKE2_ARTIFACT_PATH
# If set, the install script will use the local path for sourcing the rke2.linux-$SUFFIX and sha256sum-$ARCH.txt files
# rather than the downloading the files from the internet.
# Default is not set.
#
# info logs the given argument at info log level.
info() {
echo "[INFO] " "$@"
}
# warn logs the given argument at warn log level.
warn() {
echo "[WARN] " "$@" >&2
}
# fatal logs the given argument at fatal log level.
fatal() {
echo "[ERROR] " "$@" >&2
if [ -n "${SUFFIX}" ]; then
echo "[ALT] Please visit 'https://github.com/rancher/rke2/releases' directly and download the latest rke2.${SUFFIX}.tar.gz" >&2
fi
exit 1
}
# check_target_mountpoint return success if the target directory is on a dedicated mount point
check_target_mountpoint() {
mountpoint -q "${INSTALL_RKE2_TAR_PREFIX}"
}
# check_target_ro returns success if the target directory is read-only
check_target_ro() {
touch "${INSTALL_RKE2_TAR_PREFIX}"/.rke2-ro-test && rm -rf "${INSTALL_RKE2_TAR_PREFIX}"/.rke2-ro-test
test $? -ne 0
}
# setup_env defines needed environment variables.
setup_env() {
STORAGE_URL="https://storage.googleapis.com/rke2-ci-builds"
INSTALL_RKE2_GITHUB_URL="https://github.com/rancher/rke2"
DEFAULT_TAR_PREFIX="/usr/local"
# --- bail if we are not root ---
if [ ! $(id -u) -eq 0 ]; then
fatal "You need to be root to perform this install"
fi
# --- make sure install channel has a value
if [ -z "${INSTALL_RKE2_CHANNEL}" ]; then
INSTALL_RKE2_CHANNEL="stable"
fi
# --- make sure install type has a value
if [ -z "${INSTALL_RKE2_TYPE}" ]; then
INSTALL_RKE2_TYPE="${INSTALL_RKE2_EXEC:-server}"
fi
# --- use yum install method if available by default
if [ -z "${INSTALL_RKE2_ARTIFACT_PATH}" ] && [ -z "${INSTALL_RKE2_COMMIT}" ] && [ -z "${INSTALL_RKE2_METHOD}" ] && command -v yum >/dev/null 2>&1; then
INSTALL_RKE2_METHOD="yum"
fi
# --- install tarball to /usr/local by default, except if /usr/local is on a separate partition or is read-only
# --- in which case we go into /opt/rke2.
if [ -z "${INSTALL_RKE2_TAR_PREFIX}" ]; then
INSTALL_RKE2_TAR_PREFIX=${DEFAULT_TAR_PREFIX}
if check_target_mountpoint || check_target_ro; then
INSTALL_RKE2_TAR_PREFIX="/opt/rke2"
warn "${DEFAULT_TAR_PREFIX} is read-only or a mount point; installing to ${INSTALL_RKE2_TAR_PREFIX}"
fi
fi
if [ -z "${INSTALL_RKE2_AGENT_IMAGES_DIR}" ]; then
INSTALL_RKE2_AGENT_IMAGES_DIR="/var/lib/rancher/rke2/agent/images"
fi
}
# check_method_conflict will exit with an error if the user attempts to install
# via tar method on a host with existing RPMs.
check_method_conflict() {
case ${INSTALL_RKE2_METHOD} in
yum | rpm | dnf)
return
;;
*)
if rpm -q rke2-common >/dev/null 2>&1; then
fatal "Cannot perform ${INSTALL_RKE2_METHOD:-tar} install on host with existing RKE2 RPMs - please run rke2-uninstall.sh first"
fi
;;
esac
}
# setup_arch set arch and suffix,
# fatal if architecture not supported.
setup_arch() {
case ${ARCH:=$(uname -m)} in
amd64)
ARCH=amd64
SUFFIX=$(uname -s | tr '[:upper:]' '[:lower:]')-${ARCH}
;;
x86_64)
ARCH=amd64
SUFFIX=$(uname -s | tr '[:upper:]' '[:lower:]')-${ARCH}
;;
*)
fatal "unsupported architecture ${ARCH}"
;;
esac
}
# verify_downloader verifies existence of
# network downloader executable.
verify_downloader() {
cmd="$(command -v "${1}")"
if [ -z "${cmd}" ]; then
return 1
fi
if [ ! -x "${cmd}" ]; then
return 1
fi
# Set verified executable as our downloader program and return success
DOWNLOADER=${cmd}
return 0
}
# setup_tmp creates a temporary directory
# and cleans up when done.
setup_tmp() {
TMP_DIR=$(mktemp -d -t rke2-install.XXXXXXXXXX)
TMP_CHECKSUMS=${TMP_DIR}/rke2.checksums
TMP_TARBALL=${TMP_DIR}/rke2.tarball
TMP_AIRGAP_CHECKSUMS=${TMP_DIR}/rke2-images.checksums
TMP_AIRGAP_TARBALL=${TMP_DIR}/rke2-images.tarball
cleanup() {
code=$?
set +e
trap - EXIT
rm -rf "${TMP_DIR}"
exit $code
}
trap cleanup INT EXIT
}
# --- use desired rke2 version if defined or find version from channel ---
get_release_version() {
if [ -n "${INSTALL_RKE2_COMMIT}" ]; then
version="commit ${INSTALL_RKE2_COMMIT}"
elif [ -n "${INSTALL_RKE2_VERSION}" ]; then
version=${INSTALL_RKE2_VERSION}
else
info "finding release for channel ${INSTALL_RKE2_CHANNEL}"
INSTALL_RKE2_CHANNEL_URL=${INSTALL_RKE2_CHANNEL_URL:-'https://update.rke2.io/v1-release/channels'}
version_url="${INSTALL_RKE2_CHANNEL_URL}/${INSTALL_RKE2_CHANNEL}"
case ${DOWNLOADER} in
*curl)
version=$(${DOWNLOADER} -w "%{url_effective}" -L -s -S "${version_url}" -o /dev/null | sed -e 's|.*/||')
;;
*wget)
version=$(${DOWNLOADER} -SqO /dev/null "${version_url}" 2>&1 | grep -i Location | sed -e 's|.*/||')
;;
*)
fatal "Unsupported downloader executable '${DOWNLOADER}'"
;;
esac
INSTALL_RKE2_VERSION="${version}"
fi
}
# check_download performs a HEAD request to see if a file exists at a given url
check_download() {
case ${DOWNLOADER} in
*curl)
curl -o "/dev/null" -fsLI -X HEAD "$1"
;;
*wget)
wget -q --spider "$1"
;;
*)
fatal "downloader executable not supported: '${DOWNLOADER}'"
;;
esac
}
# download downloads a file from a url using either curl or wget
download() {
if [ $# -ne 2 ]; then
fatal "download needs exactly 2 arguments"
fi
case ${DOWNLOADER} in
*curl)
curl -o "$1" -fsSL "$2"
;;
*wget)
wget -qO "$1" "$2"
;;
*)
fatal "downloader executable not supported: '${DOWNLOADER}'"
;;
esac
# Abort if download command failed
if [ $? -ne 0 ]; then
fatal "download failed"
fi
}
# download_checksums downloads hash from github url.
download_checksums() {
if [ -n "${INSTALL_RKE2_COMMIT}" ]; then
CHECKSUMS_URL=${STORAGE_URL}/rke2.${SUFFIX}-${INSTALL_RKE2_COMMIT}.tar.gz.sha256sum
else
CHECKSUMS_URL=${INSTALL_RKE2_GITHUB_URL}/releases/download/${INSTALL_RKE2_VERSION}/sha256sum-${ARCH}.txt
fi
info "downloading checksums at ${CHECKSUMS_URL}"
download "${TMP_CHECKSUMS}" "${CHECKSUMS_URL}"
CHECKSUM_EXPECTED=$(grep "rke2.${SUFFIX}.tar.gz" "${TMP_CHECKSUMS}" | awk '{print $1}')
}
# download_tarball downloads binary from github url.
download_tarball() {
if [ -n "${INSTALL_RKE2_COMMIT}" ]; then
TARBALL_URL=${STORAGE_URL}/rke2.${SUFFIX}-${INSTALL_RKE2_COMMIT}.tar.gz
else
TARBALL_URL=${INSTALL_RKE2_GITHUB_URL}/releases/download/${INSTALL_RKE2_VERSION}/rke2.${SUFFIX}.tar.gz
fi
info "downloading tarball at ${TARBALL_URL}"
download "${TMP_TARBALL}" "${TARBALL_URL}"
}
# stage_local_checksums stages the local checksum hash for validation.
stage_local_checksums() {
info "staging local checksums from ${INSTALL_RKE2_ARTIFACT_PATH}/sha256sum-${ARCH}.txt"
cp -f "${INSTALL_RKE2_ARTIFACT_PATH}/sha256sum-${ARCH}.txt" "${TMP_CHECKSUMS}"
CHECKSUM_EXPECTED=$(grep "rke2.${SUFFIX}.tar.gz" "${TMP_CHECKSUMS}" | awk '{print $1}')
if [ -f "${INSTALL_RKE2_ARTIFACT_PATH}/rke2-images.${SUFFIX}.tar.zst" ]; then
AIRGAP_CHECKSUM_EXPECTED=$(grep "rke2-images.${SUFFIX}.tar.zst" "${TMP_CHECKSUMS}" | awk '{print $1}')
elif [ -f "${INSTALL_RKE2_ARTIFACT_PATH}/rke2-images.${SUFFIX}.tar.gz" ]; then
AIRGAP_CHECKSUM_EXPECTED=$(grep "rke2-images.${SUFFIX}.tar.gz" "${TMP_CHECKSUMS}" | awk '{print $1}')
fi
}
# stage_local_tarball stages the local tarball.
stage_local_tarball() {
info "staging tarball from ${INSTALL_RKE2_ARTIFACT_PATH}/rke2.${SUFFIX}.tar.gz"
cp -f "${INSTALL_RKE2_ARTIFACT_PATH}/rke2.${SUFFIX}.tar.gz" "${TMP_TARBALL}"
}
# stage_local_airgap_tarball stages the local checksum hash for validation.
stage_local_airgap_tarball() {
if [ -f "${INSTALL_RKE2_ARTIFACT_PATH}/rke2-images.${SUFFIX}.tar.zst" ]; then
info "staging zst airgap image tarball from ${INSTALL_RKE2_ARTIFACT_PATH}/rke2-images.${SUFFIX}.tar.zst"
cp -f "${INSTALL_RKE2_ARTIFACT_PATH}/rke2-images.${SUFFIX}.tar.zst" "${TMP_AIRGAP_TARBALL}"
AIRGAP_TARBALL_FORMAT=zst
elif [ -f "${INSTALL_RKE2_ARTIFACT_PATH}/rke2-images.${SUFFIX}.tar.gz" ]; then
info "staging gzip airgap image tarball from ${INSTALL_RKE2_ARTIFACT_PATH}/rke2-images.${SUFFIX}.tar.gz"
cp -f "${INSTALL_RKE2_ARTIFACT_PATH}/rke2-images.${SUFFIX}.tar.gz" "${TMP_AIRGAP_TARBALL}"
AIRGAP_TARBALL_FORMAT=gz
fi
}
# verify_tarball verifies the downloaded installer checksum.
verify_tarball() {
info "verifying tarball"
CHECKSUM_ACTUAL=$(sha256sum "${TMP_TARBALL}" | awk '{print $1}')
if [ "${CHECKSUM_EXPECTED}" != "${CHECKSUM_ACTUAL}" ]; then
fatal "download sha256 does not match ${CHECKSUM_EXPECTED}, got ${CHECKSUM_ACTUAL}"
fi
}
# unpack_tarball extracts the tarball, correcting paths and moving systemd units as necessary
unpack_tarball() {
info "unpacking tarball file to ${INSTALL_RKE2_TAR_PREFIX}"
mkdir -p ${INSTALL_RKE2_TAR_PREFIX}
tar xzf "${TMP_TARBALL}" -C "${INSTALL_RKE2_TAR_PREFIX}"
if [ "${INSTALL_RKE2_TAR_PREFIX}" != "${DEFAULT_TAR_PREFIX}" ]; then
info "updating tarball contents to reflect install path"
sed -i "s|${DEFAULT_TAR_PREFIX}|${INSTALL_RKE2_TAR_PREFIX}|" ${INSTALL_RKE2_TAR_PREFIX}/lib/systemd/system/rke2-*.service ${INSTALL_RKE2_TAR_PREFIX}/bin/rke2-uninstall.sh
info "moving systemd units to /etc/systemd/system"
mv -f ${INSTALL_RKE2_TAR_PREFIX}/lib/systemd/system/rke2-*.service /etc/systemd/system/
info "install complete; you may want to run: export PATH=\$PATH:${INSTALL_RKE2_TAR_PREFIX}/bin"
fi
}
# download_airgap_checksums downloads the checksum file for the airgap image tarball
# and prepares the checksum value for later validation.
download_airgap_checksums() {
if [ -z "${INSTALL_RKE2_COMMIT}" ]; then
return
fi
AIRGAP_CHECKSUMS_URL=${STORAGE_URL}/rke2-images.${SUFFIX}-${INSTALL_RKE2_COMMIT}.tar.zst.sha256sum
# try for zst first; if that fails use gz for older release branches
if ! check_download "${AIRGAP_CHECKSUMS_URL}"; then
AIRGAP_CHECKSUMS_URL=${STORAGE_URL}/rke2-images.${SUFFIX}-${INSTALL_RKE2_COMMIT}.tar.gz.sha256sum
fi
info "downloading airgap checksums at ${AIRGAP_CHECKSUMS_URL}"
download "${TMP_AIRGAP_CHECKSUMS}" "${AIRGAP_CHECKSUMS_URL}"
AIRGAP_CHECKSUM_EXPECTED=$(grep "rke2-images.${SUFFIX}.tar" "${TMP_AIRGAP_CHECKSUMS}" | awk '{print $1}')
}
# download_airgap_tarball downloads the airgap image tarball.
download_airgap_tarball() {
if [ -z "${INSTALL_RKE2_COMMIT}" ]; then
return
fi
AIRGAP_TARBALL_URL=${STORAGE_URL}/rke2-images.${SUFFIX}-${INSTALL_RKE2_COMMIT}.tar.zst
# try for zst first; if that fails use gz for older release branches
if ! check_download "${AIRGAP_TARBALL_URL}"; then
AIRGAP_TARBALL_URL=${STORAGE_URL}/rke2-images.${SUFFIX}-${INSTALL_RKE2_COMMIT}.tar.gz
fi
info "downloading airgap tarball at ${AIRGAP_TARBALL_URL}"
download "${TMP_AIRGAP_TARBALL}" "${AIRGAP_TARBALL_URL}"
}
# verify_airgap_tarball compares the airgap image tarball checksum to the value
# calculated by CI when the file was uploaded.
verify_airgap_tarball() {
if [ -z "${AIRGAP_CHECKSUM_EXPECTED}" ]; then
return
fi
info "verifying airgap tarball"
AIRGAP_CHECKSUM_ACTUAL=$(sha256sum "${TMP_AIRGAP_TARBALL}" | awk '{print $1}')
if [ "${AIRGAP_CHECKSUM_EXPECTED}" != "${AIRGAP_CHECKSUM_ACTUAL}" ]; then
fatal "download sha256 does not match ${AIRGAP_CHECKSUM_EXPECTED}, got ${AIRGAP_CHECKSUM_ACTUAL}"
fi
}
# install_airgap_tarball moves the airgap image tarball into place.
install_airgap_tarball() {
if [ -z "${AIRGAP_CHECKSUM_EXPECTED}" ]; then
return
fi
mkdir -p "${INSTALL_RKE2_AGENT_IMAGES_DIR}"
# releases that provide zst artifacts can read from the compressed archive; older releases
# that produce only gzip artifacts need to have the tarball decompressed ahead of time
if grep -qF '.tar.zst' "${TMP_AIRGAP_CHECKSUMS}" || [ "${AIRGAP_TARBALL_FORMAT}" = "zst" ]; then
info "installing airgap tarball to ${INSTALL_RKE2_AGENT_IMAGES_DIR}"
mv -f "${TMP_AIRGAP_TARBALL}" "${INSTALL_RKE2_AGENT_IMAGES_DIR}/rke2-images.${SUFFIX}.tar.zst"
else
info "decompressing airgap tarball to ${INSTALL_RKE2_AGENT_IMAGES_DIR}"
gzip -dc "${TMP_AIRGAP_TARBALL}" > "${INSTALL_RKE2_AGENT_IMAGES_DIR}/rke2-images.${SUFFIX}.tar"
fi
}
# do_install_rpm builds a yum repo config from the channel and version to be installed,
# and calls yum to install the required packates.
do_install_rpm() {
maj_ver="7"
if [ -r /etc/redhat-release ] || [ -r /etc/centos-release ] || [ -r /etc/oracle-release ]; then
dist_version="$(. /etc/os-release && echo "$VERSION_ID")"
maj_ver=$(echo "$dist_version" | sed -E -e "s/^([0-9]+)\.?[0-9]*$/\1/")
case ${maj_ver} in
7|8)
:
;;
*) # In certain cases, like installing on Fedora, maj_ver will end up being something that is not 7 or 8
maj_ver="7"
;;
esac
fi
case "${INSTALL_RKE2_CHANNEL}" in
v*.*)
# We are operating with a version-based channel, so we should parse our version out
rke2_majmin=$(echo "${INSTALL_RKE2_CHANNEL}" | sed -E -e "s/^v([0-9]+\.[0-9]+).*/\1/")
rke2_rpm_channel=$(echo "${INSTALL_RKE2_CHANNEL}" | sed -E -e "s/^v[0-9]+\.[0-9]+-(.*)/\1/")
# If our regex fails to capture a "sane" channel out of the specified channel, fall back to `stable`
if [ "${rke2_rpm_channel}" = ${INSTALL_RKE2_CHANNEL} ]; then
info "using stable RPM repositories"
rke2_rpm_channel="stable"
fi
;;
*)
get_release_version
rke2_majmin=$(echo "${INSTALL_RKE2_VERSION}" | sed -E -e "s/^v([0-9]+\.[0-9]+).*/\1/")
rke2_rpm_channel=${1}
;;
esac
info "using ${rke2_majmin} series from channel ${rke2_rpm_channel}"
rpm_site="rpm.rancher.io"
if [ "${rke2_rpm_channel}" = "testing" ]; then
rpm_site="rpm-${rke2_rpm_channel}.rancher.io"
fi
rm -f /etc/yum.repos.d/rancher-rke2*.repo
cat <<-EOF >"/etc/yum.repos.d/rancher-rke2.repo"
[rancher-rke2-common-${rke2_rpm_channel}]
name=Rancher RKE2 Common (${1})
baseurl=https://${rpm_site}/rke2/${rke2_rpm_channel}/common/centos/${maj_ver}/noarch
enabled=1
gpgcheck=1
gpgkey=https://${rpm_site}/public.key
[rancher-rke2-${rke2_majmin}-${rke2_rpm_channel}]
name=Rancher RKE2 ${rke2_majmin} (${1})
baseurl=https://${rpm_site}/rke2/${rke2_rpm_channel}/${rke2_majmin}/centos/${maj_ver}/x86_64
enabled=1
gpgcheck=1
gpgkey=https://${rpm_site}/public.key
EOF
if [ -z "${INSTALL_RKE2_VERSION}" ]; then
yum -y install "rke2-${INSTALL_RKE2_TYPE}"
else
rke2_rpm_version=$(echo "${INSTALL_RKE2_VERSION}" | sed -E -e "s/[\+-]/~/g" | sed -E -e "s/v(.*)/\1/")
if [ -n "${INSTALL_RKE2_RPM_RELEASE_VERSION}" ]; then
yum -y install "rke2-${INSTALL_RKE2_TYPE}-${rke2_rpm_version}-${INSTALL_RKE2_RPM_RELEASE_VERSION}"
else
yum -y install "rke2-${INSTALL_RKE2_TYPE}-${rke2_rpm_version}"
fi
fi
}
do_install_tar() {
setup_tmp
if [ -n "${INSTALL_RKE2_ARTIFACT_PATH}" ]; then
stage_local_checksums
stage_local_airgap_tarball
stage_local_tarball
else
get_release_version
info "using ${INSTALL_RKE2_VERSION:-commit $INSTALL_RKE2_COMMIT} as release"
download_airgap_checksums
download_airgap_tarball
download_checksums
download_tarball
fi
verify_airgap_tarball
install_airgap_tarball
verify_tarball
unpack_tarball
systemctl daemon-reload
}
do_install() {
setup_env
check_method_conflict
setup_arch
if [ -z "${INSTALL_RKE2_ARTIFACT_PATH}" ]; then
verify_downloader curl || verify_downloader wget || fatal "can not find curl or wget for downloading files"
fi
case ${INSTALL_RKE2_METHOD} in
yum | rpm | dnf)
do_install_rpm "${INSTALL_RKE2_CHANNEL}"
;;
*)
do_install_tar "${INSTALL_RKE2_CHANNEL}"
;;
esac
}
do_install
exit 0

173
pkg/driver/k3s.go Normal file
View File

@@ -0,0 +1,173 @@
package driver
import (
"bufio"
"context"
_ "embed"
"fmt"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/imdario/mergo"
"github.com/rancherfederal/hauler/pkg/packager/images"
"io"
"k8s.io/apimachinery/pkg/runtime/schema"
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
"sigs.k8s.io/cli-utils/pkg/object"
"sigs.k8s.io/yaml"
)
const (
k3sReleaseUrl = "https://github.com/k3s-io/k3s/releases/download"
)
//go:embed embed/k3s-init.sh
var k3sInit string
type K3s struct {
Version string
Config K3sConfig
}
//TODO: Would be nice if these just pointed to k3s/pkg/cli/cmds
type K3sConfig struct {
DataDir string `json:"data-dir,omitempty"`
KubeConfig string `json:"write-kubeconfig,omitempty"`
KubeConfigMode string `json:"write-kubeconfig-mode,omitempty"`
Disable []string `json:"disable,omitempty"`
}
//NewK3s returns a new k3s driver
func NewK3s() K3s {
//TODO: Allow for configuration overrides
return K3s{
Config: K3sConfig{
DataDir: "/var/lib/rancher/k3s",
KubeConfig: "/etc/rancher/k3s/k3s.yaml",
KubeConfigMode: "0644",
Disable: []string{},
},
}
}
func (k K3s) Name() string { return "k3s" }
func (k K3s) KubeConfigPath() string { return k.Config.KubeConfig }
func (k K3s) DataPath(elem ...string) string {
base := []string{k.Config.DataDir}
return filepath.Join(append(base, elem...)...)
}
func (k K3s) WriteConfig() error {
kCfgPath := filepath.Dir(k.Config.KubeConfig)
if err := os.MkdirAll(kCfgPath, os.ModePerm); err != nil {
return err
}
data, err := yaml.Marshal(k.Config)
c := make(map[string]interface{})
if err := yaml.Unmarshal(data, &c); err != nil {
return err
}
var uc map[string]interface{}
path := filepath.Join(kCfgPath, "config.yaml")
if data, err := os.ReadFile(path); err != nil {
err := yaml.Unmarshal(data, &uc)
if err != nil {
return err
}
}
//Merge with user defined configs taking precedence
if err := mergo.Merge(&c, uc); err != nil {
return err
}
mergedData, err := yaml.Marshal(&c)
if err != nil {
return err
}
return os.WriteFile(path, mergedData, 0644)
}
func (k K3s) Images(ctx context.Context) (map[name.Reference]v1.Image, error) {
imgs, err := k.listImages()
if err != nil {
return nil, err
}
return images.ResolveRemoteRefs(imgs...)
}
func (k K3s) Binary() (io.ReadCloser, error) {
u, err := url.Parse(fmt.Sprintf("%s/%s/%s", k3sReleaseUrl, k.Version, k.Name()))
if err != nil {
return nil, err
}
resp, err := http.Get(u.String())
if err != nil || resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to return executable for k3s %s from %s", k.Version, u.String())
}
return resp.Body, nil
}
//SystemObjects returns a slice of object.ObjMetadata required for driver to be functional and accept new resources
//hauler's bootstrapping sequence will always wait for SystemObjects to be in a Ready status before proceeding
func (k K3s) SystemObjects() (objs []object.ObjMetadata) {
for _, dep := range []string{"coredns"} {
objMeta, _ := object.CreateObjMetadata("kube-system", dep, schema.GroupKind{Kind: "Deployment", Group: "apps"})
objs = append(objs, objMeta)
}
return objs
}
func (k K3s) Start(out io.Writer) error {
if err := os.WriteFile("/opt/hauler/bin/k3s-init.sh", []byte(k3sInit), 0755); err != nil {
return err
}
cmd := exec.Command("/bin/sh", "/opt/hauler/bin/k3s-init.sh")
cmd.Env = append(os.Environ(), []string{
"INSTALL_K3S_SKIP_DOWNLOAD=true",
"INSTALL_K3S_SELINUX_WARN=true",
"INSTALL_K3S_SKIP_SELINUX_RPM=true",
"INSTALL_K3S_BIN_DIR=/opt/hauler/bin",
//TODO: Provide a real dryrun option
//"INSTALL_K3S_SKIP_START=true",
}...)
cmd.Stdout = out
return cmd.Run()
}
func (k K3s) listImages() ([]string, error) {
u, err := url.Parse(fmt.Sprintf("%s/%s/k3s-images.txt", k3sReleaseUrl, k.Version))
if err != nil {
return nil, err
}
resp, err := http.Get(u.String())
if err != nil || resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to return images for k3s %s from %s", k.Version, u.String())
}
defer resp.Body.Close()
var imgs []string
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
imgs = append(imgs, scanner.Text())
}
return imgs, nil
}

View File

@@ -74,32 +74,6 @@ func ImageMapFromBundle(b *fleetapi.Bundle) (map[name.Reference]v1.Image, error)
return ResolveRemoteRefs(di...)
}
func IdentifyImages(b *fleetapi.Bundle) (discoveredImages, error) {
opts := fleetapi.BundleDeploymentOptions{
DefaultNamespace: "default",
}
m := &manifest.Manifest{Resources: b.Spec.Resources}
//TODO: I think this is right?
objs, err := helmdeployer.Template("anything", m, opts)
if err != nil {
return nil, err
}
var di discoveredImages
for _, o := range objs {
imgs, err := imageFromRuntimeObject(o.(*unstructured.Unstructured))
if err != nil {
return nil, err
}
di = append(di, imgs...)
}
return di, err
}
//ResolveRemoteRefs will return a slice of remote images resolved from their fully qualified name
func ResolveRemoteRefs(images ...string) (map[name.Reference]v1.Image, error) {
m := make(map[name.Reference]v1.Image)
@@ -137,7 +111,7 @@ var knownImagePaths = []string{
"{.spec.containers[*].image}",
}
////imageFromRuntimeObject will return any images found in known obj specs
//imageFromRuntimeObject will return any images found in known obj specs
func imageFromRuntimeObject(obj *unstructured.Unstructured) (images []string, err error) {
objData, _ := obj.MarshalJSON()

View File

@@ -2,32 +2,42 @@ package packager
import (
"context"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
fleetapi "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
"github.com/rancher/fleet/pkg/bundle"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
"github.com/rancherfederal/hauler/pkg/driver"
"github.com/rancherfederal/hauler/pkg/fs"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/packager/images"
"io"
"k8s.io/apimachinery/pkg/util/json"
"net/http"
"path/filepath"
)
type Packager interface {
Bundles(context.Context, ...string) ([]*fleetapi.Bundle, error)
Driver(context.Context, v1alpha1.Drive) error
Fleet(context.Context, v1alpha1.Fleet) error
Archive(Archiver, v1alpha1.Package, string) error
PackageBundles(context.Context, ...string) ([]*fleetapi.Bundle, error)
PackageDriver(context.Context, driver.Driver) error
PackageFleet(context.Context, v1alpha1.Fleet) error
PackageImages(context.Context, map[name.Reference]v1.Image) error
}
type pkg struct {
fs fs.PkgFs
logger log.Logger
}
//NewPackager loads a new packager given a path on disk
func NewPackager(path string) Packager {
func NewPackager(path string, logger log.Logger) Packager {
return pkg{
fs: fs.NewPkgFS(path),
fs: fs.NewPkgFS(path),
logger: logger,
}
}
@@ -44,7 +54,7 @@ func (p pkg) Archive(a Archiver, pkg v1alpha1.Package, output string) error {
return Package(a, p.fs.Path(), output)
}
func (p pkg) Bundles(ctx context.Context, path ...string) ([]*fleetapi.Bundle, error) {
func (p pkg) PackageBundles(ctx context.Context, path ...string) ([]*fleetapi.Bundle, error) {
opts := &bundle.Options{Compress: true}
var bundles []*fleetapi.Bundle
@@ -67,33 +77,41 @@ func (p pkg) Bundles(ctx context.Context, path ...string) ([]*fleetapi.Bundle, e
return bundles, nil
}
func (p pkg) Driver(ctx context.Context, d v1alpha1.Drive) error {
if err := writeURL(p.fs, d.BinURL(), "k3s"); err != nil {
return err
}
//TODO: Stop hardcoding
if err := writeURL(p.fs, "https://get.k3s.io", "k3s-init.sh"); err != nil {
return err
}
imgMap, err := images.MapImager(d)
func (p pkg) PackageDriver(ctx context.Context, d driver.Driver) error {
rc, err := d.Binary()
if err != nil {
return err
}
for ref, im := range imgMap {
err := p.fs.AddImage(ref, im)
if err != nil {
return err
}
if err := p.fs.AddBin(rc, d.Name()); err != nil {
return err
}
rc.Close()
imgMap, err := d.Images(ctx)
if err != nil {
return err
}
err = p.PackageImages(ctx, imgMap)
if err != nil {
return err
}
return nil
}
//TODO: Add this to Driver?
func (p pkg) Fleet(ctx context.Context, fl v1alpha1.Fleet) error {
func (p pkg) PackageImages(ctx context.Context, imgMap map[name.Reference]v1.Image) error {
for ref, im := range imgMap {
if err := p.fs.AddImage(ref, im); err != nil {
return err
}
}
return nil
}
//TODO: Add this to PackageDriver?
func (p pkg) PackageFleet(ctx context.Context, fl v1alpha1.Fleet) error {
imgMap, err := images.MapImager(fl)
if err != nil {
return err
@@ -116,26 +134,3 @@ func (p pkg) Fleet(ctx context.Context, fl v1alpha1.Fleet) error {
return nil
}
func writeURL(fsys fs.PkgFs, rawURL string, name string) error {
rc, err := fetchURL(rawURL)
if err != nil {
return err
}
defer rc.Close()
return fsys.AddBin(rc, name)
}
func fetchURL(rawURL string) (io.ReadCloser, error) {
resp, err := http.Get(rawURL)
if err != nil {
return nil, err
}
if resp.StatusCode != 200 {
return nil, err
}
return resp.Body, nil
}

View File

@@ -0,0 +1,36 @@
package packager
import (
"context"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
"github.com/rancherfederal/hauler/pkg/fs"
"testing"
)
func Test_pkg_driver(t *testing.T) {
type fields struct {
fs fs.PkgFs
}
type args struct {
ctx context.Context
d v1alpha1.IDriver
}
tests := []struct {
name string
fields fields
args args
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := pkg{
fs: tt.fields.fs,
}
if err := p.driver(tt.args.ctx, tt.args.d); (err != nil) != tt.wantErr {
t.Errorf("driver() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}