From accf0138f8f679da29e3bfceacb904f6f3ab41fa Mon Sep 17 00:00:00 2001 From: Somefive Date: Thu, 21 Oct 2021 17:01:29 +0800 Subject: [PATCH] Feat: apiserver cluster api (#2526) * Fix: multicluster api * Feat: add apiserver cluster api & fix codecov * Feat: add pod capacity and resourceused stat * Feat: add cloud cluster manage * Test: add test for cloud cluster * Test: pending cloud resource api * Style: refactor * Fix: apiserver e2e test * Fix: fix application usecase policy bug * Style: add returns for cluster api * Feat: add provider detail info & add cache & add rollback protection * Style: refactor * Style: add error code --- .github/workflows/apiserver-test.yaml | 16 +- cmd/core/main.go | 2 +- go.mod | 3 + go.sum | 37 ++ pkg/apiserver/clients/kubeclient.go | 12 +- pkg/apiserver/model/cluster.go | 67 ++++ pkg/apiserver/model/model.go | 14 + pkg/apiserver/rest/apis/v1/types.go | 65 ++- pkg/apiserver/rest/usecase/application.go | 4 +- pkg/apiserver/rest/usecase/cluster.go | 379 +++++++++++++++++- pkg/apiserver/rest/utils/bcode/cluster.go | 38 ++ pkg/apiserver/rest/utils/params.go | 50 +++ pkg/apiserver/rest/webservice/cluster.go | 202 +++++++++- pkg/cloudprovider/aliyun.go | 128 ++++++ pkg/cloudprovider/cluster.go | 38 ++ pkg/cloudprovider/types.go | 34 ++ pkg/multicluster/cluster_management.go | 268 +++++++++++++ pkg/multicluster/errors.go | 29 ++ pkg/multicluster/utils.go | 26 +- references/cli/cluster.go | 170 +------- test/e2e-apiserver-test/cluster_test.go | 133 ++++++ .../oam_application_test.go | 24 +- test/e2e-apiserver-test/suite_test.go | 34 +- test/e2e-apiserver-test/utils.go | 59 +++ 24 files changed, 1575 insertions(+), 257 deletions(-) create mode 100644 pkg/apiserver/model/cluster.go create mode 100644 pkg/apiserver/rest/utils/bcode/cluster.go create mode 100644 pkg/apiserver/rest/utils/params.go create mode 100644 pkg/cloudprovider/aliyun.go create mode 100644 pkg/cloudprovider/cluster.go create mode 100644 pkg/cloudprovider/types.go create mode 100644 pkg/multicluster/cluster_management.go create mode 100644 pkg/multicluster/errors.go create mode 100644 test/e2e-apiserver-test/cluster_test.go create mode 100644 test/e2e-apiserver-test/utils.go diff --git a/.github/workflows/apiserver-test.yaml b/.github/workflows/apiserver-test.yaml index 75dc769b6..25483be82 100644 --- a/.github/workflows/apiserver-test.yaml +++ b/.github/workflows/apiserver-test.yaml @@ -62,6 +62,15 @@ jobs: version: ${{ env.KIND_VERSION }} skipClusterCreation: true + - name: Setup Kind Cluster (Worker) + run: | + kind delete cluster --name worker + kind create cluster --image kindest/node:v1.18.15@sha256:5c1b980c4d0e0e8e7eb9f36f7df525d079a96169c8a8f20d8bd108c0d0889cc4 --name worker + kubectl version + kubectl cluster-info + kind get kubeconfig --name worker --internal > /tmp/worker.kubeconfig + kind get kubeconfig --name worker > /tmp/worker.client.kubeconfig + - name: Setup Kind Cluster (Hub) run: | kind delete cluster @@ -69,7 +78,7 @@ jobs: kubectl version kubectl cluster-info - - name: Load Image to kind cluster (Hub) + - name: Load Image to kind cluster run: make kind-load - name: Cleanup for e2e tests @@ -81,7 +90,10 @@ jobs: run: make unit-test-apiserver - name: Run apiserver e2e test - run: make e2e-apiserver-test + run: | + export ALIYUN_ACCESS_KEY_ID=${{ secrets.ALIYUN_ACCESS_KEY_ID }} + export ALIYUN_ACCESS_KEY_SECRET=${{ secrets.ALIYUN_ACCESS_KEY_SECRET }} + make e2e-apiserver-test - name: Stop kubevela, get profile run: make end-e2e-core diff --git a/cmd/core/main.go b/cmd/core/main.go index cf57ca530..a56da7fb1 100644 --- a/cmd/core/main.go +++ b/cmd/core/main.go @@ -187,7 +187,7 @@ func main() { // wrapper the round tripper by multi cluster rewriter if enableClusterGateway { - if err := multicluster.Initialize(restConfig); err != nil { + if _, err := multicluster.Initialize(restConfig, true); err != nil { klog.ErrorS(err, "failed to enable multicluster") os.Exit(1) } diff --git a/go.mod b/go.mod index 06936e73b..a3c9ccbf4 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,9 @@ require ( github.com/AlecAivazis/survey/v2 v2.1.1 github.com/Masterminds/sprig v2.22.0+incompatible github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 + github.com/alibabacloud-go/cs-20151215/v2 v2.4.5 + github.com/alibabacloud-go/darabonba-openapi v0.1.4 + github.com/alibabacloud-go/tea v1.1.15 github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 github.com/briandowns/spinner v1.11.1 diff --git a/go.sum b/go.sum index 52245e12a..9dd0f234a 100644 --- a/go.sum +++ b/go.sum @@ -178,7 +178,29 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alessio/shellescape v1.2.2 h1:8LnL+ncxhWT2TR00dfJRT25JWWrhkMZXneHVWnetDZg= github.com/alessio/shellescape v1.2.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cvfHR56wJVlKE= +github.com/alibabacloud-go/cs-20151215/v2 v2.4.5 h1:v7SWYM+3nCfw7L5DjstXgwSo6PLGpMYZHdNdDi6yajU= +github.com/alibabacloud-go/cs-20151215/v2 v2.4.5/go.mod h1:pIg8PCfRO6qSylVbW9BiG6q0zaYCP/aIKCCEwsuvbPg= +github.com/alibabacloud-go/darabonba-openapi v0.1.4 h1:eV4mB+45/QxWFQqghSUVO5H5Ct4c+tCaCp4c57TCTVY= +github.com/alibabacloud-go/darabonba-openapi v0.1.4/go.mod h1:j03z4XUkIC9aBj/w5Bt7H0cygmPNt5sug8NXle68+Og= +github.com/alibabacloud-go/darabonba-string v1.0.0/go.mod h1:93cTfV3vuPhhEwGGpKKqhVW4jLe7tDpo3LUM0i0g6mA= +github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68 h1:NqugFkGxx1TXSh/pBcU00Y6bljgDPaFdh5MUSeJ7e50= +github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68/go.mod h1:6pb/Qy8c+lqua8cFpEy7g39NRRqOWc3rOwAy8m5Y2BY= +github.com/alibabacloud-go/endpoint-util v1.1.0 h1:r/4D3VSw888XGaeNpP994zDUaxdgTSHBbVfZlzf6b5Q= +github.com/alibabacloud-go/endpoint-util v1.1.0/go.mod h1:O5FuCALmCKs2Ff7JFJMudHs0I5EBgecXXxZRyswlEjE= +github.com/alibabacloud-go/openapi-util v0.0.7 h1:Kt/9kicJxvq1It739psKFBi1IB9imhqGWA9g4chIbjI= +github.com/alibabacloud-go/openapi-util v0.0.7/go.mod h1:sQuElr4ywwFRlCCberQwKRFhRzIyG4QTP/P4y1CJ6Ws= +github.com/alibabacloud-go/tea v1.1.0/go.mod h1:IkGyUSX4Ba1V+k4pCtJUc6jDpZLFph9QMy2VUPTwukg= +github.com/alibabacloud-go/tea v1.1.7/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4= +github.com/alibabacloud-go/tea v1.1.8/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4= +github.com/alibabacloud-go/tea v1.1.11/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4= +github.com/alibabacloud-go/tea v1.1.15 h1:IaBC1Mm5Ss+l7cWnOXSxCmnWoWrEdeHEtDgQzoCCgjY= +github.com/alibabacloud-go/tea v1.1.15/go.mod h1:nXxjm6CIFkBhwW4FQkNrolwbfon8Svy6cujmKFUq98A= +github.com/alibabacloud-go/tea-utils v1.3.1/go.mod h1:EI/o33aBfj3hETm4RLiAxF/ThQdSngxrpF8rKUDJjPE= +github.com/alibabacloud-go/tea-utils v1.3.9 h1:TtbzxS+BXrisA7wzbAMRtlU8A2eWLg0ufm7m/Tl6fc4= +github.com/alibabacloud-go/tea-utils v1.3.9/go.mod h1:EI/o33aBfj3hETm4RLiAxF/ThQdSngxrpF8rKUDJjPE= github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/aliyun/credentials-go v1.1.2 h1:qU1vwGIBb3UJ8BwunHDRFtAhS6jnQLnde/yk0+Ih2GY= +github.com/aliyun/credentials-go v1.1.2/go.mod h1:ozcZaMR5kLM7pwtCMEpVmQ242suV6qTJya2bDq4X1Tw= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= @@ -803,6 +825,8 @@ github.com/gophercloud/gophercloud v0.10.0/go.mod h1:gmC5oQqMDOMO1t1gq5DquX/yAU8 github.com/gophercloud/gophercloud v0.11.0/go.mod h1:gmC5oQqMDOMO1t1gq5DquX/yAU808e/4mzjjDA76+Ss= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= +github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= github.com/gordonklaus/ineffassign v0.0.0-20210225214923-2e10b2664254/go.mod h1:M9mZEtGIsR1oDaZagNPNG9iq9n2HrhZ17dsXk73V3Lw= github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= @@ -971,6 +995,7 @@ github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -1446,7 +1471,10 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0= +github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -1521,6 +1549,8 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tjfoc/gmsm v1.3.2 h1:7JVkAn5bvUJ7HtU08iW6UiD+UTmJTIToHCfeFzkcCxM= +github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w= github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -1590,6 +1620,7 @@ github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDf github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.30/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -1723,10 +1754,12 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191219195013-becbf705a915/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200422194213-44a606286825/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= @@ -1944,6 +1977,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2091,6 +2125,7 @@ golang.org/x/tools v0.0.0-20200422205258-72e4a01eba43/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200603131246-cc40288be839/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -2310,6 +2345,8 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= diff --git a/pkg/apiserver/clients/kubeclient.go b/pkg/apiserver/clients/kubeclient.go index 8e3733022..92c482a53 100644 --- a/pkg/apiserver/clients/kubeclient.go +++ b/pkg/apiserver/clients/kubeclient.go @@ -18,9 +18,8 @@ package clients import ( "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" - "github.com/oam-dev/kubevela/pkg/utils/common" + "github.com/oam-dev/kubevela/pkg/multicluster" ) var kubeClient client.Client @@ -35,13 +34,10 @@ func GetKubeClient() (client.Client, error) { if kubeClient != nil { return kubeClient, nil } - conf, err := config.GetConfig() + var err error + kubeClient, err = multicluster.GetMulticlusterKubernetesClient() if err != nil { return nil, err } - k8sClient, err := client.New(conf, client.Options{Scheme: common.Scheme}) - if err != nil { - return nil, err - } - return k8sClient, nil + return kubeClient, nil } diff --git a/pkg/apiserver/model/cluster.go b/pkg/apiserver/model/cluster.go new file mode 100644 index 000000000..de043148b --- /dev/null +++ b/pkg/apiserver/model/cluster.go @@ -0,0 +1,67 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +// ProviderInfo describes the information from provider API +type ProviderInfo struct { + Name string `json:"name"` + ID string `json:"id"` + Zone string `json:"zone"` + Labels map[string]string `json:"labels"` +} + +// Cluster describes the model of cluster in apiserver +type Cluster struct { + Model + Name string `json:"name"` + Description string `json:"description"` + Icon string `json:"icon"` + Labels map[string]string `json:"labels"` + Status string `json:"status"` + Reason string `json:"reason"` + + Provider ProviderInfo `json:"provider"` + APIServerURL string `json:"apiServerURL"` + DashboardURL string `json:"dashboardURL"` + + KubeConfig string `json:"kubeConfig"` + KubeConfigSecret string `json:"kubeConfigSecret"` +} + +// TableName table name for datastore +func (c *Cluster) TableName() string { + return tableNamePrefix + "cluster" +} + +// PrimaryKey primary key for datastore +func (c *Cluster) PrimaryKey() string { + return c.Name +} + +// Index set to nil for list +func (c *Cluster) Index() map[string]string { + index := make(map[string]string) + if c.Name != "" { + index["name"] = c.Name + } + return index +} + +// DeepCopy create a copy of cluster +func (c *Cluster) DeepCopy() *Cluster { + return deepCopy(c).(*Cluster) +} diff --git a/pkg/apiserver/model/model.go b/pkg/apiserver/model/model.go index 7621eb9ef..bc8f3aacb 100644 --- a/pkg/apiserver/model/model.go +++ b/pkg/apiserver/model/model.go @@ -19,6 +19,7 @@ package model import ( "encoding/json" "fmt" + "reflect" "time" "k8s.io/apimachinery/pkg/runtime" @@ -110,3 +111,16 @@ func (m *Model) SetCreateTime(time time.Time) { func (m *Model) SetUpdateTime(time time.Time) { m.UpdateTime = time } + +func deepCopy(src interface{}) interface{} { + dst := reflect.New(reflect.TypeOf(src).Elem()) + + val := reflect.ValueOf(src).Elem() + nVal := dst.Elem() + for i := 0; i < val.NumField(); i++ { + nvField := nVal.Field(i) + nvField.Set(val.Field(i)) + } + + return dst.Interface() +} diff --git a/pkg/apiserver/rest/apis/v1/types.go b/pkg/apiserver/rest/apis/v1/types.go index 1662f4859..80dabc7be 100644 --- a/pkg/apiserver/rest/apis/v1/types.go +++ b/pkg/apiserver/rest/apis/v1/types.go @@ -22,6 +22,7 @@ import ( "github.com/oam-dev/kubevela/apis/core.oam.dev/common" "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/apiserver/model" + "github.com/oam-dev/kubevela/pkg/cloudprovider" ) // CtxKeyApplication request context key of application @@ -105,14 +106,47 @@ type AddonStatusResponse struct { Phase AddonPhase `json:"phase"` } +// AccessKeyRequest request parameters to access cloud provider +type AccessKeyRequest struct { + AccessKeyID string `json:"accessKeyID"` + AccessKeySecret string `json:"accessKeySecret"` +} + // CreateClusterRequest request parameters to create a cluster type CreateClusterRequest struct { - Name string `json:"name" validate:"name"` + Name string `json:"name" validate:"checkname"` Description string `json:"description,omitempty"` Icon string `json:"icon"` - KubeConfig string `json:"kubeConfig" validate:"required_without=kubeConfigSecret"` - KubeConfigSecret string `json:"kubeConfigSecret,omitempty" validate:"required_without=kubeConfig"` + KubeConfig string `json:"kubeConfig,omitempty" validate:"required_without=KubeConfigSecret"` + KubeConfigSecret string `json:"kubeConfigSecret,omitempty" validate:"required_without=KubeConfig"` Labels map[string]string `json:"labels,omitempty"` + DashboardURL string `json:"dashboardURL,omitempty"` +} + +// ConnectCloudClusterRequest request parameters to create a cluster from cloud cluster +type ConnectCloudClusterRequest struct { + AccessKeyID string `json:"accessKeyID"` + AccessKeySecret string `json:"accessKeySecret"` + ClusterID string `json:"clusterID"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Icon string `json:"icon"` + Labels map[string]string `json:"labels,omitempty"` +} + +// ClusterResourceInfo resource info of cluster +type ClusterResourceInfo struct { + WorkerNumber int `json:"workerNumber"` + MasterNumber int `json:"masterNumber"` + MemoryCapacity int64 `json:"memoryCapacity"` + CPUCapacity int64 `json:"cpuCapacity"` + GPUCapacity int64 `json:"gpuCapacity,omitempty"` + PodCapacity int64 `json:"podCapacity"` + MemoryUsed int64 `json:"memoryUsed"` + CPUUsed int64 `json:"cpuUsed"` + GPUUsed int64 `json:"gpuUsed,omitempty"` + PodUsed int64 `json:"podUsed"` + StorageClassList []string `json:"storageClassList,omitempty"` } // DetailClusterResponse cluster detail information model @@ -125,29 +159,30 @@ type DetailClusterResponse struct { DashboardURL string `json:"dashboardURL,omitempty"` } -// ClusterResourceInfo resource info of cluster -type ClusterResourceInfo struct { - WorkerNumber int `json:"workerNumber"` - MasterNumber int `json:"masterNumber"` - MemoryCapacity int64 `json:"memoryCapacity"` - CPUCapacity int64 `json:"cpuCapacity"` - GPUCapacity int64 `json:"gpuCapacity,omitempty"` - StorageClassList []string `json:"storageClassList,omitempty"` -} - // ListClusterResponse list cluster type ListClusterResponse struct { Clusters []ClusterBase `json:"clusters"` } +// ListCloudClusterResponse list cloud clusters +type ListCloudClusterResponse struct { + Clusters []cloudprovider.CloudCluster `json:"clusters"` + Total int `json:"total"` +} + // ClusterBase cluster base model type ClusterBase struct { Name string `json:"name"` Description string `json:"description"` Icon string `json:"icon"` Labels map[string]string `json:"labels"` - Status string `json:"status"` - Reason string `json:"reason"` + + Provider model.ProviderInfo `json:"providerInfo"` + APIServerURL string `json:"apiServerURL"` + DashboardURL string `json:"dashboardURL"` + + Status string `json:"status"` + Reason string `json:"reason"` } // ListApplicationResponse list applications by query params diff --git a/pkg/apiserver/rest/usecase/application.go b/pkg/apiserver/rest/usecase/application.go index e1cd15933..fd27978df 100644 --- a/pkg/apiserver/rest/usecase/application.go +++ b/pkg/apiserver/rest/usecase/application.go @@ -575,8 +575,8 @@ func (c *applicationUsecaseImpl) renderOAMApplication(ctx context.Context, appMo for _, entity := range policies { policy := entity.(*model.ApplicationPolicy) apolicy := v1beta1.AppPolicy{ - Name: component.Name, - Type: component.Type, + Name: policy.Name, + Type: policy.Type, } if policy.Properties != nil { apolicy.Properties = policy.Properties.RawExtension() diff --git a/pkg/apiserver/rest/usecase/cluster.go b/pkg/apiserver/rest/usecase/cluster.go index 7c54406c5..bafd67e2e 100644 --- a/pkg/apiserver/rest/usecase/cluster.go +++ b/pkg/apiserver/rest/usecase/cluster.go @@ -18,25 +18,396 @@ package usecase import ( "context" + "fmt" + "io/ioutil" + "os" + "time" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/oam-dev/kubevela/pkg/apiserver/clients" "github.com/oam-dev/kubevela/pkg/apiserver/datastore" + "github.com/oam-dev/kubevela/pkg/apiserver/log" + "github.com/oam-dev/kubevela/pkg/apiserver/model" apis "github.com/oam-dev/kubevela/pkg/apiserver/rest/apis/v1" + utils2 "github.com/oam-dev/kubevela/pkg/apiserver/rest/utils" + "github.com/oam-dev/kubevela/pkg/apiserver/rest/utils/bcode" + "github.com/oam-dev/kubevela/pkg/cloudprovider" + "github.com/oam-dev/kubevela/pkg/multicluster" + "github.com/oam-dev/kubevela/pkg/utils" ) // ClusterUsecase cluster manage type ClusterUsecase interface { + ListKubeClusters(context.Context, string, int, int) (*apis.ListClusterResponse, error) CreateKubeCluster(context.Context, apis.CreateClusterRequest) (*apis.ClusterBase, error) + GetKubeCluster(context.Context, string) (*apis.DetailClusterResponse, error) + ModifyKubeCluster(context.Context, apis.CreateClusterRequest, string) (*apis.ClusterBase, error) + DeleteKubeCluster(context.Context, string) (*apis.ClusterBase, error) + + ListCloudClusters(context.Context, string, apis.AccessKeyRequest, int, int) (*apis.ListCloudClusterResponse, error) + ConnectCloudCluster(context.Context, string, apis.ConnectCloudClusterRequest) (*apis.ClusterBase, error) } type clusterUsecaseImpl struct { - ds datastore.DataStore + ds datastore.DataStore + caches map[string]*utils2.MemoryCache + k8sClient client.Client } // NewClusterUsecase new cluster usecase func NewClusterUsecase(ds datastore.DataStore) ClusterUsecase { - return &clusterUsecaseImpl{ds: ds} + k8sClient, err := clients.GetKubeClient() + if err != nil { + log.Logger.Fatalf("get k8sClient failure: %s", err.Error()) + } + return &clusterUsecaseImpl{ds: ds, k8sClient: k8sClient, caches: make(map[string]*utils2.MemoryCache)} } -func (c *clusterUsecaseImpl) CreateKubeCluster(context.Context, apis.CreateClusterRequest) (*apis.ClusterBase, error) { - return nil, nil +func (c *clusterUsecaseImpl) getClusterFromDataStore(ctx context.Context, clusterName string) (*model.Cluster, error) { + cluster := &model.Cluster{ + Name: clusterName, + } + if err := c.ds.Get(ctx, cluster); err != nil { + return nil, err + } + return cluster, nil +} + +func (c *clusterUsecaseImpl) rollbackAddedClusterInDataStore(ctx context.Context, cluster *model.Cluster) { + if e := c.ds.Delete(ctx, cluster); e != nil { + log.Logger.Errorf("failed to rollback added cluster %s in data store: %s", cluster.Name, e.Error()) + } +} + +func (c *clusterUsecaseImpl) rollbackDeletedClusterInDataStore(ctx context.Context, cluster *model.Cluster) { + if e := c.ds.Add(ctx, cluster); e != nil { + log.Logger.Errorf("failed to rollback deleted cluster %s in data store: %s", cluster.Name, e.Error()) + } +} + +func (c *clusterUsecaseImpl) rollbackJoinedKubeCluster(ctx context.Context, cluster *model.Cluster) { + if e := multicluster.DetachCluster(ctx, c.k8sClient, cluster.Name); e != nil { + log.Logger.Errorf("failed to rollback joined cluster %s in kubevela: %s", cluster.Name, e.Error()) + } +} + +func (c *clusterUsecaseImpl) rollbackDetachedKubeCluster(ctx context.Context, cluster *model.Cluster) { + if _, e := joinClusterByKubeConfigString(ctx, c.k8sClient, cluster.Name, cluster.KubeConfig); e != nil { + log.Logger.Errorf("failed to rollback detached cluster %s in kubevela: %s", cluster.Name, e.Error()) + } +} + +func (c *clusterUsecaseImpl) ListKubeClusters(ctx context.Context, query string, page int, pageSize int) (*apis.ListClusterResponse, error) { + // TODO: Fuzzy query + clusters, err := c.ds.List(ctx, &model.Cluster{}, &datastore.ListOptions{Page: page, PageSize: pageSize}) + if err != nil { + return nil, errors.Wrapf(err, "failed to list cluster with query %s in data store", query) + } + resp := &apis.ListClusterResponse{ + Clusters: []apis.ClusterBase{}, + } + for _, raw := range clusters { + cluster, ok := raw.(*model.Cluster) + if ok { + resp.Clusters = append(resp.Clusters, *newClusterBaseFromCluster(cluster)) + } + } + return resp, nil +} + +func joinClusterByKubeConfigString(ctx context.Context, k8sClient client.Client, clusterName string, kubeConfig string) (string, error) { + tmpFileName := fmt.Sprintf("/tmp/cluster-secret-%s-%s-%d.kubeconfig", clusterName, utils.RandomString(8), time.Now().UnixNano()) + if err := ioutil.WriteFile(tmpFileName, []byte(kubeConfig), 0600); err != nil { + return "", errors.Wrapf(err, "failed to write kubeconfig to temp file %s", tmpFileName) + } + defer func() { + _ = os.Remove(tmpFileName) + }() + cluster, err := multicluster.JoinClusterByKubeConfig(ctx, k8sClient, tmpFileName, clusterName) + if err != nil { + if errors.Is(err, multicluster.ErrClusterExists) { + return "", bcode.ErrClusterExistsInKubernetes + } + return "", errors.Wrapf(err, "failed to join cluster") + } + return cluster.Server, nil +} + +func createClusterModelFromRequest(req apis.CreateClusterRequest, oldCluster *model.Cluster) (newCluster *model.Cluster) { + if oldCluster != nil { + newCluster = oldCluster.DeepCopy() + } else { + newCluster = &model.Cluster{} + } + newCluster.Name = req.Name + newCluster.Description = req.Description + newCluster.Icon = req.Icon + newCluster.Labels = req.Labels + newCluster.KubeConfig = req.KubeConfig + newCluster.KubeConfigSecret = req.KubeConfigSecret + newCluster.DashboardURL = req.DashboardURL + return newCluster +} + +func (c *clusterUsecaseImpl) createKubeCluster(ctx context.Context, req apis.CreateClusterRequest, providerCluster *cloudprovider.CloudCluster) (*apis.ClusterBase, error) { + var err error + cluster := createClusterModelFromRequest(req, nil) + t := time.Now() + cluster.SetCreateTime(t) + cluster.SetUpdateTime(t) + if providerCluster != nil { + cluster.Provider = model.ProviderInfo{ + Name: providerCluster.Name, + ID: providerCluster.ID, + Zone: providerCluster.Zone, + Labels: providerCluster.Labels, + } + cluster.DashboardURL = providerCluster.DashBoardURL + } + if req.KubeConfig != "" { + cluster.APIServerURL, err = joinClusterByKubeConfigString(ctx, c.k8sClient, req.Name, req.KubeConfig) + if err != nil { + return nil, err + } + c.setClusterStatusAndResourceInfo(ctx, cluster) + if err = c.ds.Add(ctx, cluster); err != nil { + c.rollbackJoinedKubeCluster(ctx, cluster) + if errors.Is(err, datastore.ErrRecordExist) { + return nil, bcode.ErrClusterAlreadyExistInDataStore + } + return nil, err + } + return newClusterBaseFromCluster(cluster), nil + } + if req.KubeConfigSecret != "" { + return nil, bcode.ErrKubeConfigSecretNotSupport + } + return nil, bcode.ErrKubeConfigAndSecretIsNotSet +} + +func (c *clusterUsecaseImpl) CreateKubeCluster(ctx context.Context, req apis.CreateClusterRequest) (*apis.ClusterBase, error) { + return c.createKubeCluster(ctx, req, nil) +} + +func (c *clusterUsecaseImpl) GetKubeCluster(ctx context.Context, clusterName string) (*apis.DetailClusterResponse, error) { + cluster, err := c.getClusterFromDataStore(ctx, clusterName) + if err != nil { + if errors.Is(err, datastore.ErrRecordNotExist) { + return nil, bcode.ErrClusterNotFoundInDataStore + } + return nil, errors.Wrapf(err, "failed to found cluster %s in data store", clusterName) + } + resourceInfo := c.setClusterStatusAndResourceInfo(ctx, cluster) + if err = c.ds.Put(ctx, cluster); err != nil { + return nil, errors.Wrapf(err, "failed to update cluster %s status info", clusterName) + } + return &apis.DetailClusterResponse{ + ClusterBase: *newClusterBaseFromCluster(cluster), + ResourceInfo: resourceInfo, + RemoteManageURL: "NA", + DashboardURL: "NA", + }, nil +} + +func (c *clusterUsecaseImpl) ModifyKubeCluster(ctx context.Context, req apis.CreateClusterRequest, clusterName string) (*apis.ClusterBase, error) { + oldCluster, err := c.getClusterFromDataStore(ctx, clusterName) + if err != nil { + if errors.Is(err, datastore.ErrRecordNotExist) { + return nil, bcode.ErrClusterNotFoundInDataStore + } + return nil, errors.Wrapf(err, "failed to found cluster %s in data store", clusterName) + } + + newCluster := createClusterModelFromRequest(req, oldCluster) + newCluster.SetUpdateTime(time.Now()) + if oldCluster.Name != newCluster.Name || oldCluster.KubeConfig != newCluster.KubeConfig || oldCluster.KubeConfigSecret != newCluster.KubeConfigSecret { + if newCluster.KubeConfig == "" && newCluster.KubeConfigSecret != "" { + return nil, bcode.ErrKubeConfigSecretNotSupport + } + newClusterTempName := newCluster.Name + "_tmp_" + utils.RandomString(8) + newCluster.APIServerURL, err = joinClusterByKubeConfigString(ctx, c.k8sClient, newCluster.Name, newCluster.KubeConfig) + if err != nil { + return nil, errors.Wrapf(err, "failed to join new cluster %s", newCluster.Name) + } + c.setClusterStatusAndResourceInfo(ctx, newCluster) + rollbackTempCluster := func() { + rollBackCluster := newCluster.DeepCopy() + rollBackCluster.Name = newClusterTempName + c.rollbackJoinedKubeCluster(ctx, rollBackCluster) + } + if err = multicluster.DetachCluster(ctx, c.k8sClient, oldCluster.Name); err != nil { + rollbackTempCluster() + return nil, errors.Wrapf(err, "failed to detach old cluster %s", oldCluster.Name) + } + if err = c.ds.Delete(ctx, oldCluster); err != nil { + rollbackTempCluster() + c.rollbackDetachedKubeCluster(ctx, oldCluster) + if errors.Is(err, datastore.ErrRecordNotExist) { + return nil, bcode.ErrClusterNotFoundInDataStore + } + return nil, errors.Wrapf(err, "failed to delete old cluster %s from datastore", oldCluster.Name) + } + if err = c.ds.Add(ctx, newCluster); err != nil { + rollbackTempCluster() + c.rollbackDetachedKubeCluster(ctx, oldCluster) + c.rollbackDeletedClusterInDataStore(ctx, oldCluster) + if errors.Is(err, datastore.ErrRecordExist) { + return nil, bcode.ErrClusterAlreadyExistInDataStore + } + return nil, errors.Wrapf(err, "failed to add new cluster %s to datastore", newCluster.Name) + } + if err = multicluster.RenameCluster(ctx, c.k8sClient, newClusterTempName, newCluster.Name); err != nil { + rollbackTempCluster() + c.rollbackDetachedKubeCluster(ctx, oldCluster) + c.rollbackDeletedClusterInDataStore(ctx, oldCluster) + c.rollbackAddedClusterInDataStore(ctx, newCluster) + return nil, errors.Wrapf(err, "failed to rename temporary cluster %s to %s", newClusterTempName, newCluster.Name) + } + } else { + newCluster.Status = oldCluster.Status + newCluster.Reason = oldCluster.Reason + if err = c.ds.Put(ctx, newCluster); err != nil { + return nil, errors.Wrapf(err, "failed to update cluster %s", newCluster.Name) + } + } + return newClusterBaseFromCluster(newCluster), nil +} + +func (c *clusterUsecaseImpl) DeleteKubeCluster(ctx context.Context, clusterName string) (*apis.ClusterBase, error) { + cluster, err := c.getClusterFromDataStore(ctx, clusterName) + if err != nil { + if errors.Is(err, datastore.ErrRecordNotExist) { + return nil, bcode.ErrClusterNotFoundInDataStore + } + return nil, errors.Wrapf(err, "failed to found cluster %s in data store", clusterName) + } + if err = c.ds.Delete(ctx, cluster); err != nil { + if errors.Is(err, datastore.ErrRecordNotExist) { + return nil, bcode.ErrClusterNotFoundInDataStore + } + return nil, errors.Wrapf(err, "failed to delete cluster %s in data store", clusterName) + } + if err = multicluster.DetachCluster(ctx, c.k8sClient, clusterName); err != nil { + c.rollbackDeletedClusterInDataStore(ctx, cluster) + return nil, errors.Wrapf(err, "failed to delete cluster %s in kubernetes", clusterName) + } + return newClusterBaseFromCluster(cluster), nil +} + +func (c *clusterUsecaseImpl) setClusterStatusAndResourceInfo(ctx context.Context, cluster *model.Cluster) apis.ClusterResourceInfo { + resourceInfo, err := c.getClusterResourceInfoFromK8s(ctx, cluster.Name) + if err != nil { + cluster.Status = "Unhealthy" + cluster.Reason = fmt.Sprintf("Failed to get cluster resource info: %s", err.Error()) + } else { + cluster.Status = "Healthy" + cluster.Reason = "" + } + return resourceInfo +} + +func (c *clusterUsecaseImpl) getClusterResourceInfoCacheKey(clusterName string) string { + return "cluster-resource-info::" + clusterName +} + +func (c *clusterUsecaseImpl) getClusterResourceInfoFromK8s(ctx context.Context, clusterName string) (apis.ClusterResourceInfo, error) { + cacheKey := c.getClusterResourceInfoCacheKey(clusterName) + if cache, exists := c.caches[cacheKey]; exists && !cache.IsExpired() { + return cache.GetData().(apis.ClusterResourceInfo), nil + } + clusterInfo, err := multicluster.GetClusterInfo(ctx, c.k8sClient, clusterName) + if err != nil { + return apis.ClusterResourceInfo{}, err + } + var storageClassList []string + for _, cls := range clusterInfo.StorageClasses.Items { + storageClassList = append(storageClassList, cls.Name) + } + getUsed := func(cap resource.Quantity, alloc resource.Quantity) *resource.Quantity { + used := cap.DeepCopy() + used.Sub(alloc) + return &used + } + // TODO add support for gpu capacity + clusterResourceInfo := apis.ClusterResourceInfo{ + WorkerNumber: clusterInfo.WorkerNumber, + MasterNumber: clusterInfo.MasterNumber, + MemoryCapacity: clusterInfo.MemoryCapacity.Value(), + CPUCapacity: clusterInfo.CPUCapacity.Value(), + GPUCapacity: 0, + PodCapacity: clusterInfo.PodCapacity.Value(), + MemoryUsed: getUsed(clusterInfo.MemoryCapacity, clusterInfo.MemoryAllocatable).Value(), + CPUUsed: getUsed(clusterInfo.CPUCapacity, clusterInfo.CPUAllocatable).Value(), + GPUUsed: 0, + PodUsed: getUsed(clusterInfo.PodCapacity, clusterInfo.PodAllocatable).Value(), + StorageClassList: storageClassList, + } + c.caches[cacheKey] = utils2.NewMemoryCache(clusterResourceInfo, time.Minute) + return clusterResourceInfo, nil +} + +func (c *clusterUsecaseImpl) ListCloudClusters(ctx context.Context, provider string, req apis.AccessKeyRequest, pageNumber int, pageSize int) (*apis.ListCloudClusterResponse, error) { + p, err := cloudprovider.GetClusterProvider(provider, req.AccessKeyID, req.AccessKeySecret) + if err != nil { + log.Logger.Errorf("failed to get cluster provider: %s", err.Error()) + return nil, bcode.ErrInvalidCloudClusterProvider + } + clusters, total, err := p.ListCloudClusters(pageNumber, pageSize) + if err != nil { + log.Logger.Errorf("failed to list cloud clusters: %s", err.Error()) + return nil, bcode.ErrGetCloudClusterFailure + } + resp := &apis.ListCloudClusterResponse{ + Clusters: []cloudprovider.CloudCluster{}, + Total: total, + } + for _, cluster := range clusters { + resp.Clusters = append(resp.Clusters, *cluster) + } + return resp, nil +} + +func (c *clusterUsecaseImpl) ConnectCloudCluster(ctx context.Context, provider string, req apis.ConnectCloudClusterRequest) (*apis.ClusterBase, error) { + p, err := cloudprovider.GetClusterProvider(provider, req.AccessKeyID, req.AccessKeySecret) + if err != nil { + log.Logger.Errorf("failed to get cluster provider: %s", err.Error()) + return nil, bcode.ErrInvalidCloudClusterProvider + } + kubeConfig, err := p.GetClusterKubeConfig(req.ClusterID) + if err != nil { + log.Logger.Errorf("failed to get cluster kubeConfig: %s", err.Error()) + return nil, bcode.ErrGetCloudClusterFailure + } + cluster, err := p.GetClusterInfo(req.ClusterID) + if err != nil { + log.Logger.Errorf("failed to get cluster info: %s", err.Error()) + return nil, bcode.ErrGetCloudClusterFailure + } + createReq := apis.CreateClusterRequest{ + Name: req.Name, + Description: req.Description, + Icon: req.Icon, + Labels: req.Labels, + KubeConfig: kubeConfig, + } + return c.createKubeCluster(ctx, createReq, cluster) +} + +func newClusterBaseFromCluster(cluster *model.Cluster) *apis.ClusterBase { + return &apis.ClusterBase{ + Name: cluster.Name, + Description: cluster.Description, + Icon: cluster.Icon, + Labels: cluster.Labels, + + APIServerURL: cluster.APIServerURL, + DashboardURL: cluster.DashboardURL, + Provider: cluster.Provider, + + Status: cluster.Status, + Reason: cluster.Reason, + } } diff --git a/pkg/apiserver/rest/utils/bcode/cluster.go b/pkg/apiserver/rest/utils/bcode/cluster.go new file mode 100644 index 000000000..463513c71 --- /dev/null +++ b/pkg/apiserver/rest/utils/bcode/cluster.go @@ -0,0 +1,38 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bcode + +// ErrInvalidCloudClusterProvider provider is not support now +var ErrInvalidCloudClusterProvider = NewBcode(400, 40000, "provider is not support") + +// ErrKubeConfigSecretNotSupport kubeConfig secret is not support +var ErrKubeConfigSecretNotSupport = NewBcode(400, 40001, "kubeConfig secret is not supported now") + +// ErrKubeConfigAndSecretIsNotSet kubeConfig and kubeConfigSecret are not set +var ErrKubeConfigAndSecretIsNotSet = NewBcode(400, 40002, "kubeConfig or kubeConfig secret must be provided") + +// ErrClusterNotFoundInDataStore cluster not found in datastore +var ErrClusterNotFoundInDataStore = NewBcode(404, 40003, "cluster not found in data store") + +// ErrClusterAlreadyExistInDataStore cluster exists in datastore +var ErrClusterAlreadyExistInDataStore = NewBcode(400, 40004, "cluster already exists in data store") + +// ErrGetCloudClusterFailure get cloud cluster failed +var ErrGetCloudClusterFailure = NewBcode(500, 40005, "get cloud cluster information failed") + +// ErrClusterExistsInKubernetes cluster exists in kubernetes +var ErrClusterExistsInKubernetes = NewBcode(400, 40006, "cluster already exists in kubernetes") diff --git a/pkg/apiserver/rest/utils/params.go b/pkg/apiserver/rest/utils/params.go new file mode 100644 index 000000000..aff95f34d --- /dev/null +++ b/pkg/apiserver/rest/utils/params.go @@ -0,0 +1,50 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "strconv" + + "github.com/emicklei/go-restful/v3" + "github.com/pkg/errors" +) + +// ExtractPagingParams extract `page` and `pageSize` params from request +func ExtractPagingParams(req *restful.Request, minPageSize int, maxPageSize int) (int, int, error) { + pageStr := req.QueryParameter("page") + pageSizeStr := req.QueryParameter("pageSize") + page64, err := strconv.ParseInt(pageStr, 10, 32) + if err != nil { + return 0, 0, errors.Errorf("invalid page %s: %v", pageStr, err) + } + pageSize64, err := strconv.ParseInt(pageSizeStr, 10, 32) + if err != nil { + return 0, 0, errors.Errorf("invalid pageSize %s: %v", pageSizeStr, err) + } + page := int(page64) + pageSize := int(pageSize64) + if page < 0 { + page = 0 + } + if pageSize < minPageSize { + pageSize = minPageSize + } + if pageSize > maxPageSize { + pageSize = maxPageSize + } + return page, pageSize, nil +} diff --git a/pkg/apiserver/rest/webservice/cluster.go b/pkg/apiserver/rest/webservice/cluster.go index 9d6fed598..1f4f3e548 100644 --- a/pkg/apiserver/rest/webservice/cluster.go +++ b/pkg/apiserver/rest/webservice/cluster.go @@ -22,6 +22,7 @@ import ( apis "github.com/oam-dev/kubevela/pkg/apiserver/rest/apis/v1" "github.com/oam-dev/kubevela/pkg/apiserver/rest/usecase" + "github.com/oam-dev/kubevela/pkg/apiserver/rest/utils" "github.com/oam-dev/kubevela/pkg/apiserver/rest/utils/bcode" ) @@ -45,39 +46,92 @@ func (c *ClusterWebService) GetWebService() *restful.WebService { tags := []string{"cluster"} - ws.Route(ws.GET("/").To(noop). + ws.Route(ws.GET("/").To(c.listKubeClusters). Doc("list all clusters"). Metadata(restfulspec.KeyOpenAPITags, tags). Param(ws.QueryParameter("query", "Fuzzy search based on name or description").DataType("string")). + Param(ws.QueryParameter("page", "Page for paging").DataType("int").DefaultValue("0")). + Param(ws.QueryParameter("pageSize", "PageSize for paging").DataType("int").DefaultValue("20")). + Returns(200, "", apis.ListClusterResponse{}). + Returns(400, "", bcode.Bcode{}). Writes(apis.ListClusterResponse{}).Do(returns200, returns500)) ws.Route(ws.POST("/").To(c.createKubeCluster). Doc("create cluster"). Metadata(restfulspec.KeyOpenAPITags, tags). Reads(&apis.CreateClusterRequest{}). + Returns(200, "", apis.ClusterBase{}). + Returns(400, "", bcode.Bcode{}). Writes(apis.ClusterBase{})) - ws.Route(ws.GET("/{clusterName}").To(noop). + ws.Route(ws.GET("/{clusterName}").To(c.getKubeCluster). Doc("detail cluster info"). Metadata(restfulspec.KeyOpenAPITags, tags). Param(ws.PathParameter("clusterName", "identifier of the cluster").DataType("string")). + Returns(200, "", apis.DetailClusterResponse{}). + Returns(400, "", bcode.Bcode{}). Writes(apis.DetailClusterResponse{})) - // Do not implement this dimension for now. - // ws.Route(ws.GET("/{clusterName}/addons").To(noop). - // Doc("list cluster addons info"). - // Metadata(restfulspec.KeyOpenAPITags, tags). - // Param(ws.PathParameter("clusterName", "identifier of the cluster").DataType("string")). - // Writes(apis.ListClusterAddonResponse{})) + ws.Route(ws.POST("/{clusterName}").To(c.modifyKubeCluster). + Doc("modify cluster"). + Metadata(restfulspec.KeyOpenAPITags, tags). + Param(ws.PathParameter("clusterName", "identifier of the cluster").DataType("string")). + Reads(&apis.CreateClusterRequest{}). + Returns(200, "", apis.ClusterBase{}). + Returns(400, "", bcode.Bcode{}). + Writes(apis.ClusterBase{})) + + ws.Route(ws.DELETE("/{clusterName}").To(c.deleteKubeCluster). + Doc("delete cluster"). + Metadata(restfulspec.KeyOpenAPITags, tags). + Param(ws.PathParameter("clusterName", "identifier of the cluster").DataType("string")). + Returns(200, "", apis.ClusterBase{}). + Returns(400, "", bcode.Bcode{}). + Writes(apis.ClusterBase{})) + + ws.Route(ws.POST("/cloud-clusters/{provider}").To(c.listCloudClusters). + Doc("list cloud clusters"). + Metadata(restfulspec.KeyOpenAPITags, tags). + Param(ws.QueryParameter("page", "Page for paging").DataType("int").DefaultValue("0")). + Param(ws.QueryParameter("pageSize", "PageSize for paging").DataType("int").DefaultValue("20")). + Reads(&apis.AccessKeyRequest{}). + Returns(200, "", apis.ListCloudClusterResponse{}). + Returns(400, "", bcode.Bcode{}). + Writes(apis.ListCloudClusterResponse{})) + + ws.Route(ws.POST("/cloud-clusters/{provider}/connect").To(c.connectCloudCluster). + Doc("create cluster from cloud cluster"). + Metadata(restfulspec.KeyOpenAPITags, tags). + Reads(&apis.ConnectCloudClusterRequest{}). + Returns(200, "", apis.ClusterBase{}). + Returns(400, "", bcode.Bcode{}). + Writes(apis.ClusterBase{})) - // ws.Route(ws.POST("/{clusterName}/addons").To(noop). - // Doc("add addon for the cluster"). - // Metadata(restfulspec.KeyOpenAPITags, tags). - // Param(ws.PathParameter("clusterName", "identifier of the cluster").DataType("string")). - // Writes(apis.DeatilClusterAddonResponse{}).Returns(200, "", apis.DeatilClusterAddonResponse{})) return ws } +func (c *ClusterWebService) listKubeClusters(req *restful.Request, res *restful.Response) { + query := req.QueryParameter("query") + page, pageSize, err := utils.ExtractPagingParams(req, 5, 100) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + + // Call the usecase layer code + clusters, err := c.clusterUsecase.ListKubeClusters(req.Request.Context(), query, page, pageSize) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + + // Write back response data + if err := res.WriteEntity(clusters); err != nil { + bcode.ReturnError(req, res, err) + return + } +} + func (c *ClusterWebService) createKubeCluster(req *restful.Request, res *restful.Response) { // Verify the validity of parameters var createReq apis.CreateClusterRequest @@ -102,3 +156,125 @@ func (c *ClusterWebService) createKubeCluster(req *restful.Request, res *restful return } } + +func (c *ClusterWebService) getKubeCluster(req *restful.Request, res *restful.Response) { + clusterName := req.PathParameter("clusterName") + + // Call the usecase layer code + clusterDetail, err := c.clusterUsecase.GetKubeCluster(req.Request.Context(), clusterName) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + + // Write back response data + if err := res.WriteEntity(clusterDetail); err != nil { + bcode.ReturnError(req, res, err) + return + } +} + +func (c *ClusterWebService) modifyKubeCluster(req *restful.Request, res *restful.Response) { + // Verify the validity of parameters + var createReq apis.CreateClusterRequest + if err := req.ReadEntity(&createReq); err != nil { + bcode.ReturnError(req, res, err) + return + } + if err := validate.Struct(&createReq); err != nil { + bcode.ReturnError(req, res, err) + return + } + clusterName := req.PathParameter("clusterName") + + // Call the usecase layer code + clusterBase, err := c.clusterUsecase.ModifyKubeCluster(req.Request.Context(), createReq, clusterName) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + + // Write back response data + if err := res.WriteEntity(clusterBase); err != nil { + bcode.ReturnError(req, res, err) + return + } +} + +func (c *ClusterWebService) deleteKubeCluster(req *restful.Request, res *restful.Response) { + clusterName := req.PathParameter("clusterName") + + // Call the usecase layer code + clusterBase, err := c.clusterUsecase.DeleteKubeCluster(req.Request.Context(), clusterName) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + + // Write back response data + if err := res.WriteEntity(clusterBase); err != nil { + bcode.ReturnError(req, res, err) + return + } +} + +func (c *ClusterWebService) listCloudClusters(req *restful.Request, res *restful.Response) { + provider := req.PathParameter("provider") + page, pageSize, err := utils.ExtractPagingParams(req, 5, 100) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + + // Verify the validity of parameters + var accessKeyRequest apis.AccessKeyRequest + if err := req.ReadEntity(&accessKeyRequest); err != nil { + bcode.ReturnError(req, res, err) + return + } + if err := validate.Struct(&accessKeyRequest); err != nil { + bcode.ReturnError(req, res, err) + return + } + + // Call the usecase layer code + clustersResp, err := c.clusterUsecase.ListCloudClusters(req.Request.Context(), provider, accessKeyRequest, page, pageSize) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + + // Write back response data + if err := res.WriteEntity(clustersResp); err != nil { + bcode.ReturnError(req, res, err) + return + } +} + +func (c *ClusterWebService) connectCloudCluster(req *restful.Request, res *restful.Response) { + provider := req.PathParameter("provider") + + // Verify the validity of parameters + var connectReq apis.ConnectCloudClusterRequest + if err := req.ReadEntity(&connectReq); err != nil { + bcode.ReturnError(req, res, err) + return + } + if err := validate.Struct(&connectReq); err != nil { + bcode.ReturnError(req, res, err) + return + } + + // Call the usecase layer code + cluster, err := c.clusterUsecase.ConnectCloudCluster(req.Request.Context(), provider, connectReq) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + + // Write back response data + if err := res.WriteEntity(cluster); err != nil { + bcode.ReturnError(req, res, err) + return + } +} diff --git a/pkg/cloudprovider/aliyun.go b/pkg/cloudprovider/aliyun.go new file mode 100644 index 000000000..6ccf9985b --- /dev/null +++ b/pkg/cloudprovider/aliyun.go @@ -0,0 +1,128 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudprovider + +import ( + "encoding/json" + + cs20151215 "github.com/alibabacloud-go/cs-20151215/v2/client" + openapi "github.com/alibabacloud-go/darabonba-openapi/client" + "github.com/alibabacloud-go/tea/tea" + "k8s.io/klog/v2" + "k8s.io/utils/pointer" +) + +const ( + aliyunAPIEndpoint = "cs.cn-hangzhou.aliyuncs.com" +) + +// AliyunCloudProvider describes the cloud provider in aliyun +type AliyunCloudProvider struct { + *cs20151215.Client +} + +// NewAliyunCloudProvider create aliyun cloud provider +func NewAliyunCloudProvider(accessKeyID string, accessKeySecret string) (*AliyunCloudProvider, error) { + config := &openapi.Config{ + AccessKeyId: pointer.String(accessKeyID), + AccessKeySecret: pointer.String(accessKeySecret), + } + config.Endpoint = tea.String(aliyunAPIEndpoint) + c, err := cs20151215.NewClient(config) + if err != nil { + return nil, err + } + return &AliyunCloudProvider{Client: c}, nil +} + +func (provider *AliyunCloudProvider) decodeClusterLabels(tags []*cs20151215.Tag) map[string]string { + labels := map[string]string{} + for _, tag := range tags { + labels[*tag.Key] = *tag.Value + } + return labels +} + +func (provider *AliyunCloudProvider) decodeClusterURL(masterURL string) (url struct { + APIServerEndpoint string `json:"api_server_endpoint"` + DashboardEndpoint string `json:"dashboardEndpoint"` + IntranetAPIServerEndpoint string `json:"intranet_api_server_endpoint"` +}) { + if err := json.Unmarshal([]byte(masterURL), &url); err != nil { + klog.Info("failed to unmarshal masterUrl %s", masterURL) + } + return +} + +// ListCloudClusters list clusters with page info, return clusters, total count and error +func (provider *AliyunCloudProvider) ListCloudClusters(pageNumber int, pageSize int) ([]*CloudCluster, int, error) { + describeClustersV1Request := &cs20151215.DescribeClustersV1Request{ + PageSize: pointer.Int64(int64(pageSize)), + PageNumber: pointer.Int64(int64(pageNumber)), + } + resp, err := provider.DescribeClustersV1(describeClustersV1Request) + if err != nil { + return nil, 0, err + } + var clusters []*CloudCluster + for _, cluster := range resp.Body.Clusters { + labels := provider.decodeClusterLabels(cluster.Tags) + url := provider.decodeClusterURL(*cluster.MasterUrl) + clusters = append(clusters, &CloudCluster{ + ID: *cluster.ClusterId, + Name: *cluster.Name, + Type: *cluster.ClusterType, + Zone: *cluster.ZoneId, + Labels: labels, + Status: *cluster.State, + APIServerURL: url.APIServerEndpoint, + DashBoardURL: url.DashboardEndpoint, + }) + } + return clusters, int(*resp.Body.PageInfo.TotalCount), nil +} + +// GetClusterKubeConfig get cluster kubeconfig by clusterID +func (provider *AliyunCloudProvider) GetClusterKubeConfig(clusterID string) (string, error) { + req := &cs20151215.DescribeClusterUserKubeconfigRequest{} + resp, err := provider.DescribeClusterUserKubeconfig(pointer.String(clusterID), req) + if err != nil { + return "", err + } + return *resp.Body.Config, nil +} + +// GetClusterInfo retrieves cluster info by clusterID +func (provider *AliyunCloudProvider) GetClusterInfo(clusterID string) (*CloudCluster, error) { + resp, err := provider.DescribeClusterDetail(pointer.String(clusterID)) + if err != nil { + return nil, err + } + cluster := resp.Body + labels := provider.decodeClusterLabels(cluster.Tags) + url := provider.decodeClusterURL(*cluster.MasterUrl) + return &CloudCluster{ + ID: *cluster.ClusterId, + Name: *cluster.Name, + Type: *cluster.ClusterType, + Zone: *cluster.ZoneId, + Labels: labels, + Status: *cluster.State, + APIServerURL: url.APIServerEndpoint, + DashBoardURL: url.DashboardEndpoint, + }, nil +} diff --git a/pkg/cloudprovider/cluster.go b/pkg/cloudprovider/cluster.go new file mode 100644 index 000000000..5f1c530cc --- /dev/null +++ b/pkg/cloudprovider/cluster.go @@ -0,0 +1,38 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudprovider + +import ( + "github.com/pkg/errors" +) + +// CloudClusterProvider abstracts the cloud provider to provide cluster access +type CloudClusterProvider interface { + ListCloudClusters(pageNumber int, pageSize int) ([]*CloudCluster, int, error) + GetClusterKubeConfig(clusterID string) (string, error) + GetClusterInfo(clusterID string) (*CloudCluster, error) +} + +// GetClusterProvider creates interface for getting cloud cluster provider +func GetClusterProvider(provider string, accessKeyID string, accessKeySecret string) (CloudClusterProvider, error) { + switch provider { + case ProviderAliyun: + return NewAliyunCloudProvider(accessKeyID, accessKeySecret) + default: + return nil, errors.Errorf("cluster provider %s is not implemented", provider) + } +} diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go new file mode 100644 index 000000000..319643d06 --- /dev/null +++ b/pkg/cloudprovider/types.go @@ -0,0 +1,34 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudprovider + +const ( + // ProviderAliyun cloud provider aliyun + ProviderAliyun = "aliyun" +) + +// CloudCluster describes the interface that cloud provider should return +type CloudCluster struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + Zone string `json:"zone"` + Labels map[string]string `json:"labels"` + Status string `json:"status"` + APIServerURL string `json:"apiServerURL"` + DashBoardURL string `json:"dashboardURL"` +} diff --git a/pkg/multicluster/cluster_management.go b/pkg/multicluster/cluster_management.go new file mode 100644 index 000000000..8a45de977 --- /dev/null +++ b/pkg/multicluster/cluster_management.go @@ -0,0 +1,268 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package multicluster + +import ( + "context" + "fmt" + + v1alpha12 "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + v14 "k8s.io/api/storage/v1" + v13 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + errors2 "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + v12 "k8s.io/apimachinery/pkg/apis/meta/v1" + types2 "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/clientcmd/api" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha1" + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + "github.com/oam-dev/kubevela/pkg/oam" + errors3 "github.com/oam-dev/kubevela/pkg/utils/errors" +) + +// ensureResourceTrackerCRDInstalled ensures resourcetracker to be installed in child cluster +func ensureResourceTrackerCRDInstalled(ctx context.Context, c client.Client, clusterName string) error { + remoteCtx := ContextWithClusterName(ctx, clusterName) + crdName := types2.NamespacedName{Name: "resourcetrackers." + v1beta1.Group} + if err := c.Get(remoteCtx, crdName, &v13.CustomResourceDefinition{}); err != nil { + if !errors2.IsNotFound(err) { + return errors.Wrapf(err, "failed to check resourcetracker crd in cluster %s", clusterName) + } + crd := &v13.CustomResourceDefinition{} + if err = c.Get(ctx, crdName, crd); err != nil { + return errors.Wrapf(err, "failed to get resourcetracker crd in hub cluster") + } + crd.ObjectMeta = v12.ObjectMeta{ + Name: crdName.Name, + Annotations: crd.Annotations, + Labels: crd.Labels, + } + if err = c.Create(remoteCtx, crd); err != nil { + return errors.Wrapf(err, "failed to create resourcetracker crd in cluster %s", clusterName) + } + } + return nil +} + +// ensureClusterNotExists checks if child cluster has already been joined, if joined, error is returned +func ensureClusterNotExists(ctx context.Context, c client.Client, clusterName string) error { + secret := &v1.Secret{} + err := c.Get(ctx, types2.NamespacedName{Name: clusterName, Namespace: ClusterGatewaySecretNamespace}, secret) + if err == nil { + return ErrClusterExists + } + if !errors2.IsNotFound(err) { + return errors.Wrapf(err, "failed to check duplicate cluster secret") + } + return nil +} + +// getMutableClusterSecret retrieves the cluster secret and check if any application is using the cluster +func getMutableClusterSecret(ctx context.Context, c client.Client, clusterName string) (*v1.Secret, error) { + clusterSecret := &v1.Secret{} + if err := c.Get(ctx, types2.NamespacedName{Namespace: ClusterGatewaySecretNamespace, Name: clusterName}, clusterSecret); err != nil { + return nil, errors.Wrapf(err, "failed to find target cluster secret %s", clusterName) + } + labels := clusterSecret.GetLabels() + if labels == nil || labels[v1alpha12.LabelKeyClusterCredentialType] == "" { + return nil, fmt.Errorf("invalid cluster secret %s: cluster credential type label %s is not set", clusterName, v1alpha12.LabelKeyClusterCredentialType) + } + ebs := &v1alpha1.EnvBindingList{} + if err := c.List(ctx, ebs); err != nil { + return nil, errors.Wrap(err, "failed to find EnvBindings to check clusters") + } + errs := errors3.ErrorList{} + for _, eb := range ebs.Items { + for _, decision := range eb.Status.ClusterDecisions { + if decision.Cluster == clusterName { + errs.Append(fmt.Errorf("application %s/%s (env: %s, envBinding: %s) is currently using cluster %s", eb.Namespace, eb.Labels[oam.LabelAppName], decision.Env, eb.Name, clusterName)) + } + } + } + if errs.HasError() { + return nil, errors.Wrapf(errs, "cluster %s is in use now", clusterName) + } + return clusterSecret, nil +} + +// JoinClusterByKubeConfig add child cluster by kubeconfig path, return cluster info and error +func JoinClusterByKubeConfig(_ctx context.Context, k8sClient client.Client, kubeconfigPath string, clusterName string) (*api.Cluster, error) { + config, err := clientcmd.LoadFromFile(kubeconfigPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to get kubeconfig") + } + if len(config.CurrentContext) == 0 { + return nil, fmt.Errorf("current-context is not set") + } + ctx, ok := config.Contexts[config.CurrentContext] + if !ok { + return nil, fmt.Errorf("current-context %s not found", config.CurrentContext) + } + cluster, ok := config.Clusters[ctx.Cluster] + if !ok { + return nil, fmt.Errorf("cluster %s not found", ctx.Cluster) + } + authInfo, ok := config.AuthInfos[ctx.AuthInfo] + if !ok { + return nil, fmt.Errorf("authInfo %s not found", ctx.AuthInfo) + } + + if clusterName == "" { + clusterName = ctx.Cluster + } + if clusterName == ClusterLocalName { + return cluster, fmt.Errorf("cannot use `%s` as cluster name, it is reserved as the local cluster", ClusterLocalName) + } + + if err := ensureClusterNotExists(_ctx, k8sClient, clusterName); err != nil { + return cluster, errors.Wrapf(err, "cannot use cluster name %s", clusterName) + } + + var credentialType v1alpha12.CredentialType + data := map[string][]byte{ + "endpoint": []byte(cluster.Server), + "ca.crt": cluster.CertificateAuthorityData, + } + if len(authInfo.Token) > 0 { + credentialType = v1alpha12.CredentialTypeServiceAccountToken + data["token"] = []byte(authInfo.Token) + } else { + credentialType = v1alpha12.CredentialTypeX509Certificate + data["tls.crt"] = authInfo.ClientCertificateData + data["tls.key"] = authInfo.ClientKeyData + } + secret := &v1.Secret{ + ObjectMeta: v12.ObjectMeta{ + Name: clusterName, + Namespace: ClusterGatewaySecretNamespace, + Labels: map[string]string{ + v1alpha12.LabelKeyClusterCredentialType: string(credentialType), + }, + }, + Type: v1.SecretTypeOpaque, + Data: data, + } + + if err := k8sClient.Create(_ctx, secret); err != nil { + return cluster, errors.Wrapf(err, "failed to add cluster to kubernetes") + } + if err := ensureResourceTrackerCRDInstalled(_ctx, k8sClient, clusterName); err != nil { + _ = k8sClient.Delete(_ctx, secret) + return cluster, errors.Wrapf(err, "failed to ensure resourcetracker crd installed in cluster %s", clusterName) + } + return cluster, nil +} + +// DetachCluster detach cluster by name, if cluster is using by application, it will return error +func DetachCluster(ctx context.Context, k8sClient client.Client, clusterName string) error { + if clusterName == ClusterLocalName { + return ErrReservedLocalClusterName + } + clusterSecret, err := getMutableClusterSecret(ctx, k8sClient, clusterName) + if err != nil { + return errors.Wrapf(err, "cluster %s is not mutable now", clusterName) + } + return k8sClient.Delete(ctx, clusterSecret) +} + +// RenameCluster rename cluster +func RenameCluster(ctx context.Context, k8sClient client.Client, oldClusterName string, newClusterName string) error { + if newClusterName == ClusterLocalName { + return ErrReservedLocalClusterName + } + clusterSecret, err := getMutableClusterSecret(ctx, k8sClient, oldClusterName) + if err != nil { + return errors.Wrapf(err, "cluster %s is not mutable now", oldClusterName) + } + if err := ensureClusterNotExists(ctx, k8sClient, newClusterName); err != nil { + return errors.Wrapf(err, "cannot set cluster name to %s", newClusterName) + } + if err := k8sClient.Delete(ctx, clusterSecret); err != nil { + return errors.Wrapf(err, "failed to rename cluster from %s to %s", oldClusterName, newClusterName) + } + clusterSecret.ObjectMeta = v12.ObjectMeta{ + Name: newClusterName, + Namespace: ClusterGatewaySecretNamespace, + Labels: clusterSecret.Labels, + Annotations: clusterSecret.Annotations, + } + if err := k8sClient.Create(ctx, clusterSecret); err != nil { + return errors.Wrapf(err, "failed to rename cluster from %s to %s", oldClusterName, newClusterName) + } + return nil +} + +// ClusterInfo describes the basic information of a cluster +type ClusterInfo struct { + Nodes *v1.NodeList + WorkerNumber int + MasterNumber int + MemoryCapacity resource.Quantity + CPUCapacity resource.Quantity + PodCapacity resource.Quantity + MemoryAllocatable resource.Quantity + CPUAllocatable resource.Quantity + PodAllocatable resource.Quantity + StorageClasses *v14.StorageClassList +} + +// GetClusterInfo retrieves current cluster info from cluster +func GetClusterInfo(_ctx context.Context, k8sClient client.Client, clusterName string) (*ClusterInfo, error) { + ctx := ContextWithClusterName(_ctx, clusterName) + nodes := &v1.NodeList{} + if err := k8sClient.List(ctx, nodes); err != nil { + return nil, errors.Wrapf(err, "failed to list cluster nodes") + } + var workerNumber, masterNumber int + var memoryCapacity, cpuCapacity, podCapacity, memoryAllocatable, cpuAllocatable, podAllcatable resource.Quantity + for _, node := range nodes.Items { + if _, ok := node.Labels["node-role.kubernetes.io/master"]; ok { + masterNumber++ + } else { + workerNumber++ + } + capacity := node.Status.Capacity + memoryCapacity.Add(*capacity.Memory()) + cpuCapacity.Add(*capacity.Cpu()) + podCapacity.Add(*capacity.Pods()) + allocatable := node.Status.Allocatable + memoryAllocatable.Add(*allocatable.Memory()) + cpuAllocatable.Add(*allocatable.Cpu()) + podAllcatable.Add(*allocatable.Pods()) + } + storageClasses := &v14.StorageClassList{} + if err := k8sClient.List(ctx, storageClasses); err != nil { + return nil, errors.Wrapf(err, "failed to list storage classes") + } + return &ClusterInfo{ + Nodes: nodes, + WorkerNumber: workerNumber, + MasterNumber: masterNumber, + MemoryCapacity: memoryCapacity, + CPUCapacity: cpuCapacity, + PodCapacity: podCapacity, + MemoryAllocatable: memoryAllocatable, + CPUAllocatable: cpuAllocatable, + PodAllocatable: podAllcatable, + StorageClasses: storageClasses, + }, nil +} diff --git a/pkg/multicluster/errors.go b/pkg/multicluster/errors.go new file mode 100644 index 000000000..99b1432e9 --- /dev/null +++ b/pkg/multicluster/errors.go @@ -0,0 +1,29 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package multicluster + +import "fmt" + +var ( + // ErrClusterExists cluster already exists + ErrClusterExists = ClusterManagementError(fmt.Errorf("cluster already exists")) + // ErrReservedLocalClusterName reserved cluster name is used + ErrReservedLocalClusterName = ClusterManagementError(fmt.Errorf("cluster name `local` is reserved for kubevela hub cluster")) +) + +// ClusterManagementError multicluster management error +type ClusterManagementError error diff --git a/pkg/multicluster/utils.go b/pkg/multicluster/utils.go index 31428e53c..9f8182bdd 100644 --- a/pkg/multicluster/utils.go +++ b/pkg/multicluster/utils.go @@ -33,6 +33,7 @@ import ( "k8s.io/klog/v2" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" "github.com/oam-dev/kubevela/pkg/utils/common" errors3 "github.com/oam-dev/kubevela/pkg/utils/errors" @@ -114,23 +115,25 @@ func WaitUntilClusterGatewayReady(ctx context.Context, c client.Client, maxRetry // Initialize prepare multicluster environment by checking cluster gateway service in clusters and hack rest config to use cluster gateway // if cluster gateway service is not ready, it will wait up to 5 minutes -func Initialize(restConfig *rest.Config) error { +func Initialize(restConfig *rest.Config, autoUpgrade bool) (client.Client, error) { c, err := client.New(restConfig, client.Options{Scheme: common.Scheme}) if err != nil { - return errors2.Wrapf(err, "unable to get client to find cluster gateway service") + return nil, errors2.Wrapf(err, "unable to get client to find cluster gateway service") } svc, err := WaitUntilClusterGatewayReady(context.Background(), c, 60, 5*time.Second) if err != nil { - return errors2.Wrapf(err, "failed to wait for cluster gateway, unable to use multi-cluster") + return nil, errors2.Wrapf(err, "failed to wait for cluster gateway, unable to use multi-cluster") } ClusterGatewaySecretNamespace = svc.Namespace klog.Infof("find cluster gateway service %s/%s:%d", svc.Namespace, svc.Name, *svc.Port) restConfig.Wrap(NewSecretModeMultiClusterRoundTripper) - if err = UpgradeExistingClusterSecret(context.Background(), c); err != nil { - // this error do not affect the running of current version - klog.ErrorS(err, "error encountered while grading existing cluster secret to the latest version") + if autoUpgrade { + if err = UpgradeExistingClusterSecret(context.Background(), c); err != nil { + // this error do not affect the running of current version + klog.ErrorS(err, "error encountered while grading existing cluster secret to the latest version") + } } - return nil + return c, nil } // UpgradeExistingClusterSecret upgrade outdated cluster secrets in v1.1.1 to latest @@ -157,3 +160,12 @@ func UpgradeExistingClusterSecret(ctx context.Context, c client.Client) error { } return nil } + +// GetMulticlusterKubernetesClient get client with multicluster function enabled +func GetMulticlusterKubernetesClient() (client.Client, error) { + k8sConfig, err := config.GetConfig() + if err != nil { + return nil, err + } + return Initialize(k8sConfig, false) +} diff --git a/references/cli/cluster.go b/references/cli/cluster.go index 1e285952d..570e6df2e 100644 --- a/references/cli/cluster.go +++ b/references/cli/cluster.go @@ -18,26 +18,16 @@ package cli import ( "context" - "fmt" v1alpha12 "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1" "github.com/pkg/errors" "github.com/spf13/cobra" v1 "k8s.io/api/core/v1" - v13 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - errors2 "k8s.io/apimachinery/pkg/api/errors" - v12 "k8s.io/apimachinery/pkg/apis/meta/v1" - types2 "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha1" - "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/multicluster" - "github.com/oam-dev/kubevela/pkg/oam" "github.com/oam-dev/kubevela/pkg/utils/common" - errors3 "github.com/oam-dev/kubevela/pkg/utils/errors" "github.com/oam-dev/kubevela/references/a/preimport" ) @@ -115,42 +105,6 @@ func NewClusterListCommand(c *common.Args) *cobra.Command { return cmd } -func ensureClusterNotExists(c client.Client, clusterName string) error { - secret := &v1.Secret{} - err := c.Get(context.Background(), types2.NamespacedName{Name: clusterName, Namespace: multicluster.ClusterGatewaySecretNamespace}, secret) - if err == nil { - return fmt.Errorf("cluster %s already exists", clusterName) - } - if !errors2.IsNotFound(err) { - return errors.Wrapf(err, "failed to check duplicate cluster secret") - } - return nil -} - -func ensureResourceTrackerCRDInstalled(c client.Client, clusterName string) error { - ctx := context.Background() - remoteCtx := multicluster.ContextWithClusterName(ctx, clusterName) - crdName := types2.NamespacedName{Name: "resourcetrackers." + v1beta1.Group} - if err := c.Get(remoteCtx, crdName, &v13.CustomResourceDefinition{}); err != nil { - if !errors2.IsNotFound(err) { - return errors.Wrapf(err, "failed to check resourcetracker crd in cluster %s", clusterName) - } - crd := &v13.CustomResourceDefinition{} - if err = c.Get(ctx, crdName, crd); err != nil { - return errors.Wrapf(err, "failed to get resourcetracker crd in hub cluster") - } - crd.ObjectMeta = v12.ObjectMeta{ - Name: crdName.Name, - Annotations: crd.Annotations, - Labels: crd.Labels, - } - if err = c.Create(remoteCtx, crd); err != nil { - return errors.Wrapf(err, "failed to create resourcetracker crd in cluster %s", clusterName) - } - } - return nil -} - // NewClusterJoinCommand create command to help user join cluster to multicluster management func NewClusterJoinCommand(c *common.Args) *cobra.Command { cmd := &cobra.Command{ @@ -161,71 +115,14 @@ func NewClusterJoinCommand(c *common.Args) *cobra.Command { "> vela cluster join my-child-cluster.kubeconfig --name example-cluster", Args: cobra.ExactValidArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - config, err := clientcmd.LoadFromFile(args[0]) - if err != nil { - return errors.Wrapf(err, "failed to get kubeconfig") - } - if len(config.CurrentContext) == 0 { - return fmt.Errorf("current-context is not set") - } - ctx, ok := config.Contexts[config.CurrentContext] - if !ok { - return fmt.Errorf("current-context %s not found", config.CurrentContext) - } - cluster, ok := config.Clusters[ctx.Cluster] - if !ok { - return fmt.Errorf("cluster %s not found", ctx.Cluster) - } - authInfo, ok := config.AuthInfos[ctx.AuthInfo] - if !ok { - return fmt.Errorf("authInfo %s not found", ctx.AuthInfo) - } - // get ClusterName from flag or config clusterName, err := cmd.Flags().GetString(FlagClusterName) if err != nil { return errors.Wrapf(err, "failed to get cluster name flag") } - if clusterName == "" { - clusterName = ctx.Cluster - } - if clusterName == multicluster.ClusterLocalName { - return fmt.Errorf("cannot use `%s` as cluster name, it is reserved as the local cluster", multicluster.ClusterLocalName) - } - - if err := ensureClusterNotExists(c.Client, clusterName); err != nil { - return errors.Wrapf(err, "cannot use cluster name %s", clusterName) - } - var credentialType v1alpha12.CredentialType - data := map[string][]byte{ - "endpoint": []byte(cluster.Server), - "ca.crt": cluster.CertificateAuthorityData, - } - if len(authInfo.Token) > 0 { - credentialType = v1alpha12.CredentialTypeServiceAccountToken - data["token"] = []byte(authInfo.Token) - } else { - credentialType = v1alpha12.CredentialTypeX509Certificate - data["tls.crt"] = authInfo.ClientCertificateData - data["tls.key"] = authInfo.ClientKeyData - } - secret := &v1.Secret{ - ObjectMeta: v12.ObjectMeta{ - Name: clusterName, - Namespace: multicluster.ClusterGatewaySecretNamespace, - Labels: map[string]string{ - v1alpha12.LabelKeyClusterCredentialType: string(credentialType), - }, - }, - Type: v1.SecretTypeOpaque, - Data: data, - } - if err := c.Client.Create(context.Background(), secret); err != nil { - return errors.Wrapf(err, "failed to add cluster to kubernetes") - } - if err := ensureResourceTrackerCRDInstalled(c.Client, clusterName); err != nil { - _ = c.Client.Delete(context.Background(), secret) - return errors.Wrapf(err, "failed to ensure resourcetracker crd installed in cluster %s", clusterName) + cluster, err := multicluster.JoinClusterByKubeConfig(context.Background(), c.Client, args[0], clusterName) + if err != nil { + return err } cmd.Printf("Successfully add cluster %s, endpoint: %s.\n", clusterName, cluster.Server) return nil @@ -235,33 +132,6 @@ func NewClusterJoinCommand(c *common.Args) *cobra.Command { return cmd } -func getMutableClusterSecret(c client.Client, clusterName string) (*v1.Secret, error) { - clusterSecret := &v1.Secret{} - if err := c.Get(context.Background(), types2.NamespacedName{Namespace: multicluster.ClusterGatewaySecretNamespace, Name: clusterName}, clusterSecret); err != nil { - return nil, errors.Wrapf(err, "failed to find target cluster secret %s", clusterName) - } - labels := clusterSecret.GetLabels() - if labels == nil || labels[v1alpha12.LabelKeyClusterCredentialType] == "" { - return nil, fmt.Errorf("invalid cluster secret %s: cluster credential type label %s is not set", clusterName, v1alpha12.LabelKeyClusterCredentialType) - } - ebs := &v1alpha1.EnvBindingList{} - if err := c.List(context.Background(), ebs); err != nil { - return nil, errors.Wrap(err, "failed to find EnvBindings to check clusters") - } - errs := errors3.ErrorList{} - for _, eb := range ebs.Items { - for _, decision := range eb.Status.ClusterDecisions { - if decision.Cluster == clusterName { - errs.Append(fmt.Errorf("application %s/%s (env: %s, envBinding: %s) is currently using cluster %s", eb.Namespace, eb.Labels[oam.LabelAppName], decision.Env, eb.Name, clusterName)) - } - } - } - if errs.HasError() { - return nil, errors.Wrapf(errs, "cluster %s is in use now", clusterName) - } - return clusterSecret, nil -} - // NewClusterRenameCommand create command to help user rename cluster func NewClusterRenameCommand(c *common.Args) *cobra.Command { cmd := &cobra.Command{ @@ -271,27 +141,8 @@ func NewClusterRenameCommand(c *common.Args) *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { oldClusterName := args[0] newClusterName := args[1] - if newClusterName == multicluster.ClusterLocalName { - return fmt.Errorf("cannot use `%s` as cluster name, it is reserved as the local cluster", multicluster.ClusterLocalName) - } - clusterSecret, err := getMutableClusterSecret(c.Client, oldClusterName) - if err != nil { - return errors.Wrapf(err, "cluster %s is not mutable now", oldClusterName) - } - if err := ensureClusterNotExists(c.Client, newClusterName); err != nil { - return errors.Wrapf(err, "cannot set cluster name to %s", newClusterName) - } - if err := c.Client.Delete(context.Background(), clusterSecret); err != nil { - return errors.Wrapf(err, "failed to rename cluster from %s to %s", oldClusterName, newClusterName) - } - clusterSecret.ObjectMeta = v12.ObjectMeta{ - Name: newClusterName, - Namespace: multicluster.ClusterGatewaySecretNamespace, - Labels: clusterSecret.Labels, - Annotations: clusterSecret.Annotations, - } - if err := c.Client.Create(context.Background(), clusterSecret); err != nil { - return errors.Wrapf(err, "failed to rename cluster from %s to %s", oldClusterName, newClusterName) + if err := multicluster.RenameCluster(context.Background(), c.Client, oldClusterName, newClusterName); err != nil { + return err } cmd.Printf("Rename cluster %s to %s successfully.\n", oldClusterName, newClusterName) return nil @@ -308,15 +159,8 @@ func NewClusterDetachCommand(c *common.Args) *cobra.Command { Args: cobra.ExactValidArgs(1), RunE: func(cmd *cobra.Command, args []string) error { clusterName := args[0] - if clusterName == multicluster.ClusterLocalName { - return fmt.Errorf("cannot delete `%s` cluster, it is reserved as the local cluster", multicluster.ClusterLocalName) - } - clusterSecret, err := getMutableClusterSecret(c.Client, clusterName) - if err != nil { - return errors.Wrapf(err, "cluster %s is not mutable now", clusterName) - } - if err := c.Client.Delete(context.Background(), clusterSecret); err != nil { - return errors.Wrapf(err, "failed to detach cluster %s", clusterName) + if err := multicluster.DetachCluster(context.Background(), c.Client, clusterName); err != nil { + return err } cmd.Printf("Detach cluster %s successfully.\n", clusterName) return nil diff --git a/test/e2e-apiserver-test/cluster_test.go b/test/e2e-apiserver-test/cluster_test.go new file mode 100644 index 000000000..180dbb7f1 --- /dev/null +++ b/test/e2e-apiserver-test/cluster_test.go @@ -0,0 +1,133 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_apiserver + +import ( + "io/ioutil" + "net/http" + "os" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + v1 "github.com/oam-dev/kubevela/pkg/apiserver/rest/apis/v1" + util "github.com/oam-dev/kubevela/pkg/utils" +) + +const ( + WorkerClusterName = "cluster-worker" + WorkerClusterKubeConfigPath = "/tmp/worker.kubeconfig" +) + +var _ = Describe("Test cluster rest api", func() { + + Context("Test basic cluster CURD", func() { + + var clusterName string + + BeforeEach(func() { + clusterName = WorkerClusterName + "-" + util.RandomString(8) + kubeconfigBytes, err := ioutil.ReadFile(WorkerClusterKubeConfigPath) + Expect(err).Should(Succeed()) + resp, err := CreateRequest(http.MethodPost, "/clusters", v1.CreateClusterRequest{ + Name: clusterName, + KubeConfig: string(kubeconfigBytes), + }) + Expect(err).Should(Succeed()) + Expect(resp.StatusCode).Should(Equal(200)) + Expect(resp.Body).ShouldNot(BeNil()) + Expect(resp.Body.Close()).Should(Succeed()) + }) + + AfterEach(func() { + resp, err := CreateRequest(http.MethodDelete, "/clusters/"+clusterName, nil) + Expect(err).Should(Succeed()) + Expect(resp.StatusCode).Should(Equal(200)) + Expect(resp.Body).ShouldNot(BeNil()) + Expect(resp.Body.Close()).Should(Succeed()) + }) + + It("Test get cluster", func() { + resp, err := CreateRequest(http.MethodGet, "/clusters/"+clusterName, nil) + clusterResp := &v1.DetailClusterResponse{} + Expect(DecodeResponseBody(resp, err, clusterResp)).Should(Succeed()) + Expect(clusterResp.Status).Should(Equal("Healthy")) + }) + + It("Test get clusters", func() { + resp, err := CreateRequest(http.MethodGet, "/clusters/?page=1&pageSize=5", nil) + clusterResp := &v1.ListClusterResponse{} + Expect(DecodeResponseBody(resp, err, clusterResp)).Should(Succeed()) + Expect(len(clusterResp.Clusters)).ShouldNot(Equal(0)) + }) + + It("Test modify cluster", func() { + kubeconfigBytes, err := ioutil.ReadFile(WorkerClusterKubeConfigPath) + Expect(err).Should(Succeed()) + resp, err := CreateRequest(http.MethodPost, "/clusters/"+clusterName, v1.CreateClusterRequest{ + Name: clusterName, + KubeConfig: string(kubeconfigBytes), + Description: "Example description", + }) + clusterResp := &v1.ClusterBase{} + Expect(DecodeResponseBody(resp, err, clusterResp)).Should(Succeed()) + Expect(clusterResp.Description).ShouldNot(Equal("")) + }) + + }) + + PContext("Test cloud cluster rest api", func() { + + var clusterName string + + BeforeEach(func() { + clusterName = WorkerClusterName + "-" + util.RandomString(8) + }) + + AfterEach(func() { + resp, err := CreateRequest(http.MethodDelete, "/clusters/"+clusterName, nil) + Expect(err).Should(Succeed()) + Expect(resp.StatusCode).Should(Equal(200)) + Expect(resp.Body).ShouldNot(BeNil()) + Expect(resp.Body.Close()).Should(Succeed()) + }) + + It("Test list aliyun cloud cluster and connect", func() { + AccessKeyID := os.Getenv("ALIYUN_ACCESS_KEY_ID") + AccessKeySecret := os.Getenv("ALIYUN_ACCESS_KEY_SECRET") + resp, err := CreateRequest(http.MethodPost, "/clusters/cloud-clusters/aliyun/?page=1&pageSize=5", v1.AccessKeyRequest{ + AccessKeyID: AccessKeyID, + AccessKeySecret: AccessKeySecret, + }) + clusterResp := &v1.ListCloudClusterResponse{} + Expect(DecodeResponseBody(resp, err, clusterResp)).Should(Succeed()) + Expect(len(clusterResp.Clusters)).ShouldNot(Equal(0)) + + ClusterID := clusterResp.Clusters[0].ID + resp, err = CreateRequest(http.MethodPost, "/clusters/cloud-clusters/aliyun/connect", v1.ConnectCloudClusterRequest{ + AccessKeyID: AccessKeyID, + AccessKeySecret: AccessKeySecret, + ClusterID: ClusterID, + Name: clusterName, + }) + clusterBase := &v1.ClusterBase{} + Expect(DecodeResponseBody(resp, err, clusterBase)).Should(Succeed()) + Expect(clusterBase.Status).Should(Equal("Healthy")) + }) + + }) +}) diff --git a/test/e2e-apiserver-test/oam_application_test.go b/test/e2e-apiserver-test/oam_application_test.go index 4048541df..06f11cf08 100644 --- a/test/e2e-apiserver-test/oam_application_test.go +++ b/test/e2e-apiserver-test/oam_application_test.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" "net/http" + "time" "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo" @@ -74,17 +75,18 @@ var _ = Describe("Test oam application rest api", func() { } bodyByte, err = json.Marshal(updateReq) Expect(err).Should(BeNil()) - res, err = http.Post( - fmt.Sprintf("http://127.0.0.1:8000/v1/namespaces/%s/applications/%s", namespace, appName), - "application/json", - bytes.NewBuffer(bodyByte), - ) - Expect(err).ShouldNot(HaveOccurred()) - Expect(res).ShouldNot(BeNil()) - Expect(cmp.Diff(res.StatusCode, 200)).Should(BeEmpty()) - Expect(res.Body).ShouldNot(BeNil()) - defer res.Body.Close() - + Eventually(func(g Gomega) { + res, err = http.Post( + fmt.Sprintf("http://127.0.0.1:8000/v1/namespaces/%s/applications/%s", namespace, appName), + "application/json", + bytes.NewBuffer(bodyByte), + ) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(res).ShouldNot(BeNil()) + g.Expect(cmp.Diff(res.StatusCode, 200)).Should(BeEmpty()) + g.Expect(res.Body).ShouldNot(BeNil()) + defer res.Body.Close() + }, time.Minute).Should(Succeed()) newApp := new(v1beta1.Application) Expect(k8sClient.Get(ctx, client.ObjectKey{Name: appName, Namespace: namespace}, newApp)).Should(BeNil()) Expect(newApp.Spec.Components).Should(Equal(updateReq.Components)) diff --git a/test/e2e-apiserver-test/suite_test.go b/test/e2e-apiserver-test/suite_test.go index bcc14d355..4014e9af8 100644 --- a/test/e2e-apiserver-test/suite_test.go +++ b/test/e2e-apiserver-test/suite_test.go @@ -23,50 +23,28 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "k8s.io/client-go/rest" - "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" "github.com/oam-dev/kubevela/pkg/apiserver/clients" "github.com/oam-dev/kubevela/pkg/apiserver/datastore" arest "github.com/oam-dev/kubevela/pkg/apiserver/rest" - "github.com/oam-dev/kubevela/pkg/utils/common" ) -var cfg *rest.Config var k8sClient client.Client -var testEnv *envtest.Environment func TestE2eApiserverTest(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "E2eApiserverTest Suite") } +// Suite test in e2e-apiserver-test relies on the pre-setup kubernetes environment var _ = BeforeSuite(func() { - - By("bootstrapping test environment") - - testEnv = &envtest.Environment{ - ControlPlaneStartTimeout: time.Minute * 3, - ControlPlaneStopTimeout: time.Minute, - UseExistingCluster: pointer.BoolPtr(false), - CRDDirectoryPaths: []string{"../../charts/vela-core/crds"}, - } - - By("start kube test env") - var err error - cfg, err = testEnv.Start() - Expect(err).ShouldNot(HaveOccurred()) - Expect(cfg).ToNot(BeNil()) - By("new kube client") - cfg.Timeout = time.Minute * 2 - k8sClient, err = client.New(cfg, client.Options{Scheme: common.Scheme}) + var err error + k8sClient, err = clients.GetKubeClient() Expect(err).Should(BeNil()) Expect(k8sClient).ToNot(BeNil()) By("new kube client success") - clients.SetKubeClient(k8sClient) ctx := context.Background() @@ -86,9 +64,3 @@ var _ = BeforeSuite(func() { By("api server started") time.Sleep(time.Second * 2) }) - -var _ = AfterSuite(func() { - By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).ToNot(HaveOccurred()) -}) diff --git a/test/e2e-apiserver-test/utils.go b/test/e2e-apiserver-test/utils.go new file mode 100644 index 000000000..c3725b650 --- /dev/null +++ b/test/e2e-apiserver-test/utils.go @@ -0,0 +1,59 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_apiserver + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" +) + +// CreateRequest wraps request +func CreateRequest(method string, path string, body interface{}) (*http.Response, error) { + if body == nil { + body = map[string]string{} + } + bs, err := json.Marshal(body) + if err != nil { + return nil, err + } + req, err := http.NewRequest(method, "http://127.0.0.1:8000/api/v1"+path, bytes.NewBuffer(bs)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + return http.DefaultClient.Do(req) +} + +// DecodeResponseBody decode response and close response +func DecodeResponseBody(resp *http.Response, err error, dst interface{}) error { + if err != nil { + return err + } + if resp.StatusCode != 200 { + return fmt.Errorf("response code is not 200: %d", resp.StatusCode) + } + if resp.Body == nil { + return fmt.Errorf("response body is nil") + } + err = json.NewDecoder(resp.Body).Decode(dst) + if err != nil { + return err + } + return resp.Body.Close() +}