mirror of
https://github.com/rancher/k3k.git
synced 2026-03-16 00:20:37 +00:00
Compare commits
1 Commits
copilot/ge
...
renovate/m
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
af55e2c3f9 |
18
.gitconfig
18
.gitconfig
@@ -1,18 +0,0 @@
|
||||
[core]
|
||||
autocrlf = input
|
||||
whitespace = trailing-space,space-before-tab
|
||||
|
||||
[pull]
|
||||
rebase = true
|
||||
|
||||
[push]
|
||||
default = current
|
||||
|
||||
[rebase]
|
||||
autoStash = true
|
||||
|
||||
[alias]
|
||||
st = status
|
||||
co = checkout
|
||||
br = branch
|
||||
lg = log --oneline --graph --decorate --all
|
||||
6
.github/workflows/build.yml
vendored
6
.github/workflows/build.yml
vendored
@@ -19,10 +19,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||
|
||||
- name: Run GoReleaser
|
||||
uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6
|
||||
uses: goreleaser/goreleaser-action@ec59f474b9834571250b370d4735c50f8e2d1e29 # v7
|
||||
with:
|
||||
distribution: goreleaser
|
||||
version: v2
|
||||
|
||||
2
.github/workflows/chart.yml
vendored
2
.github/workflows/chart.yml
vendored
@@ -11,7 +11,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
|
||||
2
.github/workflows/release-delete.yml
vendored
2
.github/workflows/release-delete.yml
vendored
@@ -24,7 +24,7 @@ jobs:
|
||||
run: echo "::error::Missing tag from input" && exit 1
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
|
||||
- name: Check if release is draft
|
||||
run: |
|
||||
|
||||
6
.github/workflows/release.yml
vendored
6
.github/workflows/release.yml
vendored
@@ -21,7 +21,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
run: git checkout ${{ inputs.commit }}
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
echo "CURRENT_TAG=${CURRENT_TAG}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run GoReleaser
|
||||
uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6
|
||||
uses: goreleaser/goreleaser-action@ec59f474b9834571250b370d4735c50f8e2d1e29 # v7
|
||||
with:
|
||||
distribution: goreleaser
|
||||
version: v2
|
||||
|
||||
@@ -21,12 +21,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
- uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
@@ -131,7 +131,7 @@ jobs:
|
||||
--output-dir /tmp
|
||||
|
||||
- name: Archive conformance logs
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
|
||||
if: always()
|
||||
with:
|
||||
name: conformance-${{ matrix.type }}-logs
|
||||
|
||||
10
.github/workflows/test-conformance-virtual.yaml
vendored
10
.github/workflows/test-conformance-virtual.yaml
vendored
@@ -21,12 +21,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
- uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
@@ -104,21 +104,21 @@ jobs:
|
||||
kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log
|
||||
|
||||
- name: Archive K3s logs
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
|
||||
if: always()
|
||||
with:
|
||||
name: k3s-${{ matrix.type }}-logs
|
||||
path: /tmp/k3s.log
|
||||
|
||||
- name: Archive K3k logs
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
|
||||
if: always()
|
||||
with:
|
||||
name: k3k-${{ matrix.type }}-logs
|
||||
path: /tmp/k3k.log
|
||||
|
||||
- name: Archive conformance logs
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
|
||||
if: always()
|
||||
with:
|
||||
name: conformance-${{ matrix.type }}-logs
|
||||
|
||||
16
.github/workflows/test-e2e.yaml
vendored
16
.github/workflows/test-e2e.yaml
vendored
@@ -16,12 +16,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
- uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
@@ -78,14 +78,14 @@ jobs:
|
||||
flags: e2e
|
||||
|
||||
- name: Archive k3s logs
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-k3s-logs
|
||||
path: /tmp/k3s.log
|
||||
|
||||
- name: Archive k3k logs
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-k3k-logs
|
||||
@@ -95,12 +95,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
- uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
@@ -157,14 +157,14 @@ jobs:
|
||||
flags: e2e
|
||||
|
||||
- name: Archive k3s logs
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-k3s-logs
|
||||
path: /tmp/k3s.log
|
||||
|
||||
- name: Archive k3k logs
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-k3k-logs
|
||||
|
||||
12
.github/workflows/test.yaml
vendored
12
.github/workflows/test.yaml
vendored
@@ -16,9 +16,9 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
|
||||
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
- uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
@@ -37,12 +37,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
- uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
@@ -85,14 +85,14 @@ jobs:
|
||||
flags: cli
|
||||
|
||||
- name: Archive k3s logs
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
|
||||
if: always()
|
||||
with:
|
||||
name: cli-k3s-logs
|
||||
path: /tmp/k3s.log
|
||||
|
||||
- name: Archive k3k logs
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
|
||||
if: always()
|
||||
with:
|
||||
name: cli-k3k-logs
|
||||
|
||||
4
.github/workflows/validate.yml
vendored
4
.github/workflows/validate.yml
vendored
@@ -15,10 +15,10 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
@@ -2,7 +2,6 @@ package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"maps"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
@@ -84,30 +83,11 @@ func updateNodeCapacity(ctx context.Context, logger logr.Logger, hostClient clie
|
||||
|
||||
mergedResourceLists := mergeResourceLists(resourceLists...)
|
||||
|
||||
var virtualNodeList, hostNodeList corev1.NodeList
|
||||
|
||||
if err := virtualClient.List(ctx, &virtualNodeList); err != nil {
|
||||
logger.Error(err, "error listing virtual nodes for stable capacity distribution")
|
||||
m, err := distributeQuotas(ctx, logger, virtualClient, mergedResourceLists)
|
||||
if err != nil {
|
||||
logger.Error(err, "error distributing policy quota")
|
||||
}
|
||||
|
||||
virtResourceMap := make(map[string]corev1.ResourceList)
|
||||
for _, vNode := range virtualNodeList.Items {
|
||||
virtResourceMap[vNode.Name] = corev1.ResourceList{}
|
||||
}
|
||||
|
||||
if err := hostClient.List(ctx, &hostNodeList); err != nil {
|
||||
logger.Error(err, "error listing host nodes for stable capacity distribution")
|
||||
}
|
||||
|
||||
hostResourceMap := make(map[string]corev1.ResourceList)
|
||||
|
||||
for _, hNode := range hostNodeList.Items {
|
||||
if _, ok := virtResourceMap[hNode.Name]; ok {
|
||||
hostResourceMap[hNode.Name] = hNode.Status.Allocatable
|
||||
}
|
||||
}
|
||||
|
||||
m := distributeQuotas(hostResourceMap, virtResourceMap, mergedResourceLists)
|
||||
allocatable = m[virtualNodeName]
|
||||
}
|
||||
|
||||
@@ -145,99 +125,76 @@ func mergeResourceLists(resourceLists ...corev1.ResourceList) corev1.ResourceLis
|
||||
return merged
|
||||
}
|
||||
|
||||
// distributeQuotas divides the total resource quotas among all active virtual nodes,
|
||||
// capped by each node's actual host capacity. This ensures that each virtual node
|
||||
// reports a fair share of the available resources without exceeding what its
|
||||
// underlying host node can provide.
|
||||
// distributeQuotas divides the total resource quotas evenly among all active virtual nodes.
|
||||
// This ensures that each virtual node reports a fair share of the available resources,
|
||||
// preventing the scheduler from overloading a single node.
|
||||
//
|
||||
// For each resource type the algorithm uses a multi-pass redistribution loop:
|
||||
// 1. Divide the remaining quota evenly among eligible nodes (sorted by name for
|
||||
// determinism), assigning any integer remainder to the first nodes alphabetically.
|
||||
// 2. Cap each node's share at its host allocatable capacity.
|
||||
// 3. Remove nodes that have reached their host capacity.
|
||||
// 4. If there is still unallocated quota (because some nodes were capped below their
|
||||
// even share), repeat from step 1 with the remaining quota and remaining nodes.
|
||||
//
|
||||
// The loop terminates when the quota is fully distributed or no eligible nodes remain.
|
||||
func distributeQuotas(hostResourceMap, virtResourceMap map[string]corev1.ResourceList, quotas corev1.ResourceList) map[string]corev1.ResourceList {
|
||||
resourceMap := make(map[string]corev1.ResourceList, len(virtResourceMap))
|
||||
maps.Copy(resourceMap, virtResourceMap)
|
||||
// The algorithm iterates over each resource, divides it as evenly as possible among the
|
||||
// sorted virtual nodes, and distributes any remainder to the first few nodes to ensure
|
||||
// all resources are allocated. Sorting the nodes by name guarantees a deterministic
|
||||
// distribution.
|
||||
func distributeQuotas(ctx context.Context, logger logr.Logger, virtualClient client.Client, quotas corev1.ResourceList) (map[string]corev1.ResourceList, error) {
|
||||
// List all virtual nodes to distribute the quota stably.
|
||||
var virtualNodeList corev1.NodeList
|
||||
if err := virtualClient.List(ctx, &virtualNodeList); err != nil {
|
||||
logger.Error(err, "error listing virtual nodes for stable capacity distribution, falling back to full quota")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If there are no virtual nodes, there's nothing to distribute.
|
||||
numNodes := int64(len(virtualNodeList.Items))
|
||||
if numNodes == 0 {
|
||||
logger.Info("error listing virtual nodes for stable capacity distribution, falling back to full quota")
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Sort nodes by name for a deterministic distribution of resources.
|
||||
sort.Slice(virtualNodeList.Items, func(i, j int) bool {
|
||||
return virtualNodeList.Items[i].Name < virtualNodeList.Items[j].Name
|
||||
})
|
||||
|
||||
// Initialize the resource map for each virtual node.
|
||||
resourceMap := make(map[string]corev1.ResourceList)
|
||||
for _, virtualNode := range virtualNodeList.Items {
|
||||
resourceMap[virtualNode.Name] = corev1.ResourceList{}
|
||||
}
|
||||
|
||||
// Distribute each resource type from the policy's hard quota
|
||||
for resourceName, totalQuantity := range quotas {
|
||||
_, useMilli := milliScaleResources[resourceName]
|
||||
|
||||
// eligible nodes for each distribution cycle
|
||||
var eligibleNodes []string
|
||||
|
||||
hostCap := make(map[string]int64)
|
||||
|
||||
// Populate the host nodes capacity map and the initial effective nodes
|
||||
for vn := range virtResourceMap {
|
||||
hostNodeResources := hostResourceMap[vn]
|
||||
if hostNodeResources == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
resourceQuantity, found := hostNodeResources[resourceName]
|
||||
if !found {
|
||||
// skip the node if the resource does not exist on the host node
|
||||
continue
|
||||
}
|
||||
|
||||
hostCap[vn] = resourceQuantity.Value()
|
||||
if useMilli {
|
||||
hostCap[vn] = resourceQuantity.MilliValue()
|
||||
}
|
||||
|
||||
eligibleNodes = append(eligibleNodes, vn)
|
||||
}
|
||||
|
||||
sort.Strings(eligibleNodes)
|
||||
|
||||
totalValue := totalQuantity.Value()
|
||||
if useMilli {
|
||||
// Use MilliValue for precise division, especially for resources like CPU,
|
||||
// which are often expressed in milli-units. Otherwise, use the standard Value().
|
||||
var totalValue int64
|
||||
if _, found := milliScaleResources[resourceName]; found {
|
||||
totalValue = totalQuantity.MilliValue()
|
||||
} else {
|
||||
totalValue = totalQuantity.Value()
|
||||
}
|
||||
|
||||
// Start of the distribution cycle, each cycle will distribute the quota resource
|
||||
// evenly between nodes, each node can not exceed the corresponding host node capacity
|
||||
for totalValue > 0 && len(eligibleNodes) > 0 {
|
||||
nodeNum := int64(len(eligibleNodes))
|
||||
quantityPerNode := totalValue / nodeNum
|
||||
remainder := totalValue % nodeNum
|
||||
// Calculate the base quantity of the resource to be allocated per node.
|
||||
// and the remainder that needs to be distributed among the nodes.
|
||||
//
|
||||
// For example, if totalValue is 2000 (e.g., 2 CPU) and there are 3 nodes:
|
||||
// - quantityPerNode would be 666 (2000 / 3)
|
||||
// - remainder would be 2 (2000 % 3)
|
||||
// The first two nodes would get 667 (666 + 1), and the last one would get 666.
|
||||
quantityPerNode := totalValue / numNodes
|
||||
remainder := totalValue % numNodes
|
||||
|
||||
remainingNodes := []string{}
|
||||
|
||||
for _, virtualNodeName := range eligibleNodes {
|
||||
nodeQuantity := quantityPerNode
|
||||
if remainder > 0 {
|
||||
nodeQuantity++
|
||||
remainder--
|
||||
}
|
||||
// We cap the quantity to the hostNode capacity
|
||||
nodeQuantity = min(nodeQuantity, hostCap[virtualNodeName])
|
||||
|
||||
if nodeQuantity > 0 {
|
||||
existing := resourceMap[virtualNodeName][resourceName]
|
||||
if useMilli {
|
||||
resourceMap[virtualNodeName][resourceName] = *resource.NewMilliQuantity(existing.MilliValue()+nodeQuantity, totalQuantity.Format)
|
||||
} else {
|
||||
resourceMap[virtualNodeName][resourceName] = *resource.NewQuantity(existing.Value()+nodeQuantity, totalQuantity.Format)
|
||||
}
|
||||
}
|
||||
|
||||
totalValue -= nodeQuantity
|
||||
hostCap[virtualNodeName] -= nodeQuantity
|
||||
|
||||
if hostCap[virtualNodeName] > 0 {
|
||||
remainingNodes = append(remainingNodes, virtualNodeName)
|
||||
}
|
||||
// Iterate through the sorted virtual nodes to distribute the resource.
|
||||
for _, virtualNode := range virtualNodeList.Items {
|
||||
nodeQuantity := quantityPerNode
|
||||
if remainder > 0 {
|
||||
nodeQuantity++
|
||||
remainder--
|
||||
}
|
||||
|
||||
eligibleNodes = remainingNodes
|
||||
if _, found := milliScaleResources[resourceName]; found {
|
||||
resourceMap[virtualNode.Name][resourceName] = *resource.NewMilliQuantity(nodeQuantity, totalQuantity.Format)
|
||||
} else {
|
||||
resourceMap[virtualNode.Name][resourceName] = *resource.NewQuantity(nodeQuantity, totalQuantity.Format)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return resourceMap
|
||||
return resourceMap, nil
|
||||
}
|
||||
|
||||
@@ -1,13 +1,19 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/go-logr/zapr"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"go.uber.org/zap"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func Test_distributeQuotas(t *testing.T) {
|
||||
@@ -15,56 +21,39 @@ func Test_distributeQuotas(t *testing.T) {
|
||||
err := corev1.AddToScheme(scheme)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Large allocatable so capping doesn't interfere with basic distribution tests.
|
||||
largeAllocatable := corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("100"),
|
||||
corev1.ResourceMemory: resource.MustParse("100Gi"),
|
||||
corev1.ResourcePods: resource.MustParse("1000"),
|
||||
}
|
||||
node1 := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}}
|
||||
node2 := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-2"}}
|
||||
node3 := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-3"}}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
virtResourceMap map[string]corev1.ResourceList
|
||||
hostResourceMap map[string]corev1.ResourceList
|
||||
quotas corev1.ResourceList
|
||||
want map[string]corev1.ResourceList
|
||||
name string
|
||||
virtualNodes []client.Object
|
||||
quotas corev1.ResourceList
|
||||
want map[string]corev1.ResourceList
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "no virtual nodes",
|
||||
virtResourceMap: map[string]corev1.ResourceList{},
|
||||
name: "no virtual nodes",
|
||||
virtualNodes: []client.Object{},
|
||||
quotas: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("2"),
|
||||
},
|
||||
want: map[string]corev1.ResourceList{},
|
||||
want: map[string]corev1.ResourceList{},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "no quotas",
|
||||
virtResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": {},
|
||||
"node-2": {},
|
||||
},
|
||||
hostResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": largeAllocatable,
|
||||
"node-2": largeAllocatable,
|
||||
},
|
||||
quotas: corev1.ResourceList{},
|
||||
name: "no quotas",
|
||||
virtualNodes: []client.Object{node1, node2},
|
||||
quotas: corev1.ResourceList{},
|
||||
want: map[string]corev1.ResourceList{
|
||||
"node-1": {},
|
||||
"node-2": {},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "fewer virtual nodes than host nodes",
|
||||
virtResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": {},
|
||||
"node-2": {},
|
||||
},
|
||||
hostResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": largeAllocatable,
|
||||
"node-2": largeAllocatable,
|
||||
"node-3": largeAllocatable,
|
||||
"node-4": largeAllocatable,
|
||||
},
|
||||
name: "even distribution of cpu and memory",
|
||||
virtualNodes: []client.Object{node1, node2},
|
||||
quotas: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("2"),
|
||||
corev1.ResourceMemory: resource.MustParse("4Gi"),
|
||||
@@ -79,203 +68,65 @@ func Test_distributeQuotas(t *testing.T) {
|
||||
corev1.ResourceMemory: resource.MustParse("2Gi"),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "even distribution of cpu and memory",
|
||||
virtResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": {},
|
||||
"node-2": {},
|
||||
},
|
||||
hostResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": largeAllocatable,
|
||||
"node-2": largeAllocatable,
|
||||
},
|
||||
name: "uneven distribution with remainder",
|
||||
virtualNodes: []client.Object{node1, node2, node3},
|
||||
quotas: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("2"),
|
||||
corev1.ResourceMemory: resource.MustParse("4Gi"),
|
||||
},
|
||||
want: map[string]corev1.ResourceList{
|
||||
"node-1": {
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
corev1.ResourceMemory: resource.MustParse("2Gi"),
|
||||
},
|
||||
"node-2": {
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
corev1.ResourceMemory: resource.MustParse("2Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "uneven distribution with remainder",
|
||||
virtResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": {},
|
||||
"node-2": {},
|
||||
"node-3": {},
|
||||
},
|
||||
hostResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": largeAllocatable,
|
||||
"node-2": largeAllocatable,
|
||||
"node-3": largeAllocatable,
|
||||
},
|
||||
quotas: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("2"),
|
||||
corev1.ResourceCPU: resource.MustParse("2"), // 2000m / 3 = 666m with 2m remainder
|
||||
},
|
||||
want: map[string]corev1.ResourceList{
|
||||
"node-1": {corev1.ResourceCPU: resource.MustParse("667m")},
|
||||
"node-2": {corev1.ResourceCPU: resource.MustParse("667m")},
|
||||
"node-3": {corev1.ResourceCPU: resource.MustParse("666m")},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "distribution of number resources",
|
||||
virtResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": {},
|
||||
"node-2": {},
|
||||
"node-3": {},
|
||||
},
|
||||
hostResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": largeAllocatable,
|
||||
"node-2": largeAllocatable,
|
||||
"node-3": largeAllocatable,
|
||||
},
|
||||
name: "distribution of number resources",
|
||||
virtualNodes: []client.Object{node1, node2, node3},
|
||||
quotas: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("2"),
|
||||
corev1.ResourcePods: resource.MustParse("11"),
|
||||
corev1.ResourceCPU: resource.MustParse("2"),
|
||||
corev1.ResourcePods: resource.MustParse("11"),
|
||||
corev1.ResourceSecrets: resource.MustParse("9"),
|
||||
"custom": resource.MustParse("8"),
|
||||
},
|
||||
want: map[string]corev1.ResourceList{
|
||||
"node-1": {
|
||||
corev1.ResourceCPU: resource.MustParse("667m"),
|
||||
corev1.ResourcePods: resource.MustParse("4"),
|
||||
corev1.ResourceCPU: resource.MustParse("667m"),
|
||||
corev1.ResourcePods: resource.MustParse("4"),
|
||||
corev1.ResourceSecrets: resource.MustParse("3"),
|
||||
"custom": resource.MustParse("3"),
|
||||
},
|
||||
"node-2": {
|
||||
corev1.ResourceCPU: resource.MustParse("667m"),
|
||||
corev1.ResourcePods: resource.MustParse("4"),
|
||||
corev1.ResourceCPU: resource.MustParse("667m"),
|
||||
corev1.ResourcePods: resource.MustParse("4"),
|
||||
corev1.ResourceSecrets: resource.MustParse("3"),
|
||||
"custom": resource.MustParse("3"),
|
||||
},
|
||||
"node-3": {
|
||||
corev1.ResourceCPU: resource.MustParse("666m"),
|
||||
corev1.ResourcePods: resource.MustParse("3"),
|
||||
corev1.ResourceCPU: resource.MustParse("666m"),
|
||||
corev1.ResourcePods: resource.MustParse("3"),
|
||||
corev1.ResourceSecrets: resource.MustParse("3"),
|
||||
"custom": resource.MustParse("2"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "extended resource distributed only to nodes that have it",
|
||||
virtResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": {},
|
||||
"node-2": {},
|
||||
"node-3": {},
|
||||
},
|
||||
hostResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": {
|
||||
corev1.ResourceCPU: resource.MustParse("100"),
|
||||
"nvidia.com/gpu": resource.MustParse("2"),
|
||||
},
|
||||
"node-2": {
|
||||
corev1.ResourceCPU: resource.MustParse("100"),
|
||||
},
|
||||
"node-3": {
|
||||
corev1.ResourceCPU: resource.MustParse("100"),
|
||||
"nvidia.com/gpu": resource.MustParse("4"),
|
||||
},
|
||||
},
|
||||
quotas: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("3"),
|
||||
"nvidia.com/gpu": resource.MustParse("4"),
|
||||
},
|
||||
want: map[string]corev1.ResourceList{
|
||||
"node-1": {
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
"nvidia.com/gpu": resource.MustParse("2"),
|
||||
},
|
||||
"node-2": {
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
},
|
||||
"node-3": {
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
"nvidia.com/gpu": resource.MustParse("2"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "capping at host capacity with redistribution",
|
||||
virtResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": {},
|
||||
"node-2": {},
|
||||
},
|
||||
hostResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": {
|
||||
corev1.ResourceCPU: resource.MustParse("8"),
|
||||
},
|
||||
"node-2": {
|
||||
corev1.ResourceCPU: resource.MustParse("2"),
|
||||
},
|
||||
},
|
||||
quotas: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("6"),
|
||||
},
|
||||
// Even split would be 3 each, but node-2 only has 2 CPU.
|
||||
// node-2 gets capped at 2, the remaining 1 goes to node-1.
|
||||
want: map[string]corev1.ResourceList{
|
||||
"node-1": {corev1.ResourceCPU: resource.MustParse("4")},
|
||||
"node-2": {corev1.ResourceCPU: resource.MustParse("2")},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "gpu capping with uneven host capacity",
|
||||
virtResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": {},
|
||||
"node-2": {},
|
||||
},
|
||||
hostResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": {
|
||||
"nvidia.com/gpu": resource.MustParse("6"),
|
||||
},
|
||||
"node-2": {
|
||||
"nvidia.com/gpu": resource.MustParse("1"),
|
||||
},
|
||||
},
|
||||
quotas: corev1.ResourceList{
|
||||
"nvidia.com/gpu": resource.MustParse("4"),
|
||||
},
|
||||
// Even split would be 2 each, but node-2 only has 1 GPU.
|
||||
// node-2 gets capped at 1, the remaining 1 goes to node-1.
|
||||
want: map[string]corev1.ResourceList{
|
||||
"node-1": {"nvidia.com/gpu": resource.MustParse("3")},
|
||||
"node-2": {"nvidia.com/gpu": resource.MustParse("1")},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "quota exceeds total host capacity",
|
||||
virtResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": {},
|
||||
"node-2": {},
|
||||
"node-3": {},
|
||||
},
|
||||
hostResourceMap: map[string]corev1.ResourceList{
|
||||
"node-1": {
|
||||
"nvidia.com/gpu": resource.MustParse("2"),
|
||||
},
|
||||
"node-2": {
|
||||
"nvidia.com/gpu": resource.MustParse("1"),
|
||||
},
|
||||
"node-3": {
|
||||
"nvidia.com/gpu": resource.MustParse("1"),
|
||||
},
|
||||
},
|
||||
quotas: corev1.ResourceList{
|
||||
"nvidia.com/gpu": resource.MustParse("10"),
|
||||
},
|
||||
// Total host capacity is 4, quota is 10. Each node gets its full capacity.
|
||||
want: map[string]corev1.ResourceList{
|
||||
"node-1": {"nvidia.com/gpu": resource.MustParse("2")},
|
||||
"node-2": {"nvidia.com/gpu": resource.MustParse("1")},
|
||||
"node-3": {"nvidia.com/gpu": resource.MustParse("1")},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := distributeQuotas(tt.hostResourceMap, tt.virtResourceMap, tt.quotas)
|
||||
fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.virtualNodes...).Build()
|
||||
logger := zapr.NewLogger(zap.NewNop())
|
||||
|
||||
got, gotErr := distributeQuotas(context.Background(), logger, fakeClient, tt.quotas)
|
||||
if tt.wantErr {
|
||||
assert.Error(t, gotErr)
|
||||
} else {
|
||||
assert.NoError(t, gotErr)
|
||||
}
|
||||
|
||||
assert.Equal(t, len(tt.want), len(got), "Number of nodes in result should match")
|
||||
|
||||
|
||||
@@ -78,6 +78,7 @@ var (
|
||||
k8s *kubernetes.Clientset
|
||||
k8sClient client.Client
|
||||
kubeconfigPath string
|
||||
repo string
|
||||
helmActionConfig *action.Configuration
|
||||
)
|
||||
|
||||
@@ -86,17 +87,17 @@ var _ = BeforeSuite(func() {
|
||||
|
||||
GinkgoWriter.Println("GOCOVERDIR:", os.Getenv("GOCOVERDIR"))
|
||||
|
||||
repo = os.Getenv("REPO")
|
||||
if repo == "" {
|
||||
repo = "rancher"
|
||||
}
|
||||
|
||||
_, dockerInstallEnabled := os.LookupEnv("K3K_DOCKER_INSTALL")
|
||||
|
||||
if dockerInstallEnabled {
|
||||
repo := os.Getenv("REPO")
|
||||
if repo == "" {
|
||||
repo = "rancher"
|
||||
}
|
||||
|
||||
installK3SDocker(ctx, repo+"/k3k", repo+"/k3k-kubelet")
|
||||
installK3SDocker(ctx)
|
||||
initKubernetesClient(ctx)
|
||||
installK3kChart(repo+"/k3k", repo+"/k3k-kubelet")
|
||||
installK3kChart()
|
||||
} else {
|
||||
initKubernetesClient(ctx)
|
||||
}
|
||||
@@ -110,11 +111,6 @@ func initKubernetesClient(ctx context.Context) {
|
||||
kubeconfig []byte
|
||||
)
|
||||
|
||||
logger, err := zap.NewDevelopment()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
log.SetLogger(zapr.NewLogger(logger))
|
||||
|
||||
kubeconfigPath := os.Getenv("KUBECONFIG")
|
||||
Expect(kubeconfigPath).To(Not(BeEmpty()))
|
||||
|
||||
@@ -133,6 +129,11 @@ func initKubernetesClient(ctx context.Context) {
|
||||
scheme := buildScheme()
|
||||
k8sClient, err = client.New(restcfg, client.Options{Scheme: scheme})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
logger, err := zap.NewDevelopment()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
log.SetLogger(zapr.NewLogger(logger))
|
||||
}
|
||||
|
||||
func buildScheme() *runtime.Scheme {
|
||||
@@ -146,7 +147,7 @@ func buildScheme() *runtime.Scheme {
|
||||
return scheme
|
||||
}
|
||||
|
||||
func installK3SDocker(ctx context.Context, controllerImage, kubeletImage string) {
|
||||
func installK3SDocker(ctx context.Context) {
|
||||
var (
|
||||
err error
|
||||
kubeconfig []byte
|
||||
@@ -178,15 +179,16 @@ func installK3SDocker(ctx context.Context, controllerImage, kubeletImage string)
|
||||
Expect(tmpFile.Close()).To(Succeed())
|
||||
kubeconfigPath = tmpFile.Name()
|
||||
|
||||
err = k3sContainer.LoadImages(ctx, controllerImage+":dev", kubeletImage+":dev")
|
||||
err = k3sContainer.LoadImages(ctx, repo+"/k3k:dev", repo+"/k3k-kubelet:dev")
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
DeferCleanup(os.Remove, kubeconfigPath)
|
||||
|
||||
Expect(os.Setenv("KUBECONFIG", kubeconfigPath)).To(Succeed())
|
||||
GinkgoWriter.Printf("KUBECONFIG set to: %s\n", kubeconfigPath)
|
||||
GinkgoWriter.Print(kubeconfigPath)
|
||||
GinkgoWriter.Print(string(kubeconfig))
|
||||
}
|
||||
|
||||
func installK3kChart(controllerImage, kubeletImage string) {
|
||||
func installK3kChart() {
|
||||
pwd, err := os.Getwd()
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
@@ -202,7 +204,7 @@ func installK3kChart(controllerImage, kubeletImage string) {
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
err = helmActionConfig.Init(restClientGetter, k3kNamespace, os.Getenv("HELM_DRIVER"), func(format string, v ...any) {
|
||||
GinkgoWriter.Printf("[Helm] "+format+"\n", v...)
|
||||
GinkgoWriter.Printf("helm debug: "+format+"\n", v...)
|
||||
})
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
@@ -214,17 +216,9 @@ func installK3kChart(controllerImage, kubeletImage string) {
|
||||
iCli.Wait = true
|
||||
|
||||
controllerMap, _ := k3kChart.Values["controller"].(map[string]any)
|
||||
|
||||
extraEnvArray, _ := controllerMap["extraEnv"].([]map[string]any)
|
||||
extraEnvArray = append(extraEnvArray, map[string]any{
|
||||
"name": "DEBUG",
|
||||
"value": "true",
|
||||
})
|
||||
controllerMap["extraEnv"] = extraEnvArray
|
||||
|
||||
imageMap, _ := controllerMap["image"].(map[string]any)
|
||||
maps.Copy(imageMap, map[string]any{
|
||||
"repository": controllerImage,
|
||||
"repository": repo + "/k3k",
|
||||
"tag": "dev",
|
||||
"pullPolicy": "IfNotPresent",
|
||||
})
|
||||
@@ -233,14 +227,14 @@ func installK3kChart(controllerImage, kubeletImage string) {
|
||||
sharedAgentMap, _ := agentMap["shared"].(map[string]any)
|
||||
sharedAgentImageMap, _ := sharedAgentMap["image"].(map[string]any)
|
||||
maps.Copy(sharedAgentImageMap, map[string]any{
|
||||
"repository": kubeletImage,
|
||||
"repository": repo + "/k3k-kubelet",
|
||||
"tag": "dev",
|
||||
})
|
||||
|
||||
release, err := iCli.Run(k3kChart, k3kChart.Values)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
GinkgoWriter.Printf("Helm release '%s' installed in '%s' namespace\n", release.Name, release.Namespace)
|
||||
GinkgoWriter.Printf("Release %s installed in %s namespace\n", release.Name, release.Namespace)
|
||||
}
|
||||
|
||||
func patchPVC(ctx context.Context, clientset *kubernetes.Clientset) {
|
||||
@@ -303,28 +297,36 @@ func patchPVC(ctx context.Context, clientset *kubernetes.Clientset) {
|
||||
_, err = clientset.AppsV1().Deployments(k3kNamespace).Update(ctx, k3kDeployment, metav1.UpdateOptions{})
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
Eventually(func() bool {
|
||||
GinkgoWriter.Println("Checking K3k deployment status")
|
||||
|
||||
dep, err := clientset.AppsV1().Deployments(k3kNamespace).Get(ctx, k3kDeployment.Name, metav1.GetOptions{})
|
||||
g.Expect(err).To(Not(HaveOccurred()))
|
||||
g.Expect(dep.Generation).To(Equal(dep.Status.ObservedGeneration))
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
var availableCond appsv1.DeploymentCondition
|
||||
|
||||
for _, cond := range dep.Status.Conditions {
|
||||
if cond.Type == appsv1.DeploymentAvailable {
|
||||
availableCond = cond
|
||||
break
|
||||
}
|
||||
// 1. Check if the controller has observed the latest generation
|
||||
if dep.Generation > dep.Status.ObservedGeneration {
|
||||
GinkgoWriter.Printf("K3k deployment generation: %d, observed generation: %d\n", dep.Generation, dep.Status.ObservedGeneration)
|
||||
return false
|
||||
}
|
||||
|
||||
g.Expect(availableCond.Type).To(Equal(appsv1.DeploymentAvailable))
|
||||
g.Expect(availableCond.Status).To(Equal(v1.ConditionTrue))
|
||||
// 2. Check if all replicas have been updated
|
||||
if dep.Spec.Replicas != nil && dep.Status.UpdatedReplicas < *dep.Spec.Replicas {
|
||||
GinkgoWriter.Printf("K3k deployment replicas: %d, updated replicas: %d\n", *dep.Spec.Replicas, dep.Status.UpdatedReplicas)
|
||||
return false
|
||||
}
|
||||
|
||||
// 3. Check if all updated replicas are available
|
||||
if dep.Status.AvailableReplicas < dep.Status.UpdatedReplicas {
|
||||
GinkgoWriter.Printf("K3k deployment available replicas: %d, updated replicas: %d\n", dep.Status.AvailableReplicas, dep.Status.UpdatedReplicas)
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}).
|
||||
MustPassRepeatedly(5).
|
||||
WithPolling(time.Second).
|
||||
WithTimeout(time.Second * 30).
|
||||
Should(Succeed())
|
||||
Should(BeTrue())
|
||||
}
|
||||
|
||||
var _ = AfterSuite(func() {
|
||||
|
||||
Reference in New Issue
Block a user