From 0736e85e0772966976f8528dbf3e7cdc64e1b228 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 1 Nov 2022 00:13:37 +0800 Subject: [PATCH] [Backport release-1.6] Feat: implement pipeline APIs (#4969) * add context when run pipeline Signed-off-by: Qiaozp (cherry picked from commit 72f3ad792ef6d0f7c05a89341e02a8bf2a33ee26) * Feat: implement pipeline API Signed-off-by: Qiaozp (cherry picked from commit f560c346cc4424a1c782615a42da85cb81da5663) * Extract get log logic and implement getPipelineRunLog API Signed-off-by: Qiaozp (cherry picked from commit 060c6ab9e96a01d8e1230de5d25c1f5215cd96f8) * Init and delete pipeline contexts Signed-off-by: Qiaozp (cherry picked from commit 5e96bd31066960160977397decbbd12e08bbb1b1) * fix panic Signed-off-by: Qiaozp (cherry picked from commit 51072f7947f86594b4c32854c20e2c9580a70a21) * Allow not specifying context Signed-off-by: Qiaozp (cherry picked from commit 810ddcf0bdd98dfceee8aa6f86c283da7db9f3b0) * change pipeline to path parameter Signed-off-by: Qiaozp (cherry picked from commit 3d51c0cb2db9ca364884de539bcfc3fa27553e26) * Add permission check filter Signed-off-by: Qiaozp (cherry picked from commit 688376743040f9903bdce9af66949f6424946089) * project -> projects in route Signed-off-by: Qiaozp (cherry picked from commit 1f09f3996bf6953dc07f2b778314a238ec516836) * fix route conflict Signed-off-by: Qiaozp (cherry picked from commit 7eea69683048639ebd5d1efd362adf05d3c80bf8) * Add project alias Signed-off-by: Qiaozp (cherry picked from commit b07dd72338f376c53522bed5d58c331449659f1a) * Feat: change the list pipeline API Signed-off-by: barnettZQG (cherry picked from commit bd804734b0bfa71ed59c26613c9ecb2094a0a3d5) * Feat: filter the project Signed-off-by: barnettZQG (cherry picked from commit 82eee2cc113862bcd6db589875dfdc99d7e2bf69) * Fix: the error of the run APi Signed-off-by: barnettZQG (cherry picked from commit ac87bd3f1a2eeaf5caafe6cfb8d7e7e3992d2a2b) * fix log pipeline run API Signed-off-by: Qiaozp (cherry picked from commit acde8e981e9003a81ffa91f9e72776f29e275654) * Fix lint, fix the error of log api Signed-off-by: Qiaozp (cherry picked from commit b8373e6cde596557976fc1d26feca4badd8fd264) * fix error returning Signed-off-by: Qiaozp (cherry picked from commit 2e9b4792b06cac89e6a42f6d64c80eb09a30431d) * Fix: change the lable to annotation Signed-off-by: barnettZQG (cherry picked from commit bf08275fde2f1aa7dd7c0a80ebb2ec74ced61617) * remove log config not found error Signed-off-by: Qiaozp (cherry picked from commit cdd77dfd8fa5f32e3622c139fd51cead550742ed) * fix pipeline list api return no context info Signed-off-by: Qiaozp (cherry picked from commit cdcfa165d16d355d0a6210357e4f13e678c620d9) * Fix: create the namespace Signed-off-by: barnettZQG (cherry picked from commit b6888dd87d7c2032c448ef77f570a5db57507409) * get pipeline lastrun info Signed-off-by: Qiaozp (cherry picked from commit a943423d22c7bbbe3cfee1d98a7a4516258f6895) * allow query single step output Signed-off-by: Qiaozp (cherry picked from commit e2310bbf34f20063917b6bba7999aa2328bc4a5c) * organize code in api layer Signed-off-by: Qiaozp (cherry picked from commit 6fd53ed078e6d3dd965e86e32c41abea50c50a44) * fix project filter, add context value when get pp run, extend lastRun Signed-off-by: Qiaozp (cherry picked from commit 861f69d555fc443a153c7e14c28897f40773b8b4) * fix get output and implement get input api Signed-off-by: Qiaozp (cherry picked from commit 61495ee70dbcd928f4749807adc635fad3b1f857) * Fix: change the last run Signed-off-by: barnettZQG (cherry picked from commit aeb842a45ea7c8a81aa75770a5e54ec1eb5cdaed) * if query sub-step outout, return it directly Signed-off-by: Qiaozp (cherry picked from commit 467ba257511662d3a688b2a27489dd7939e151ef) * Fix: change the run stats Signed-off-by: barnettZQG (cherry picked from commit 7a90e7e310181f5ec4ca60be1f7f4cb14b4e0a68) * Fix: change the output Signed-off-by: barnettZQG (cherry picked from commit 595a871b0d3aa39cf939cee1ffe6dbefa78dbf57) * flatten the input/output api Signed-off-by: Qiaozp (cherry picked from commit efc9692354c21408c61f26e55d07a223778efa1e) * more info for i/o vars Signed-off-by: Qiaozp (cherry picked from commit 7fe0e1109cd0544d19cf432e885fab579490d9bf) * fix nested i/o struct Signed-off-by: Qiaozp (cherry picked from commit 201d1228bdc9e74ac85909f9fd05ce7cb5ed6441) * add fromStep in input api Signed-off-by: Qiaozp (cherry picked from commit 240001896289f7d9261bd590b17b9c6e75801cbb) * add e2e test skeleton Signed-off-by: Qiaozp (cherry picked from commit f20f9a1ac67bbdc078a8a941809e207014d5c145) * add more e2e test Signed-off-by: Qiaozp (cherry picked from commit 98b27f886b0db09509381f12b76948f74567dcaa) * use db to store pipeline Signed-off-by: Qiaozp (cherry picked from commit 08962c4f2f61ea09e3cb9a40f8f92a723e5e7309) * keep the last 5k lines of log Signed-off-by: Qiaozp (cherry picked from commit 22b352da146b42465c790fcb927889f10f0cad0d) * use stern param to keep last lines of logs Signed-off-by: Qiaozp (cherry picked from commit 3eadbf91c818092d02948bbe4798360121e6079e) * filter, nil labels, spec check Signed-off-by: Qiaozp (cherry picked from commit bad90b3f7a343647b39af2a8dd6dd8b101562dab) * empty res, index, detail param Signed-off-by: Qiaozp (cherry picked from commit b12d889e978b4f0936a83c76010701c94e314323) * Add e2e test Signed-off-by: Qiaozp (cherry picked from commit ae617a928ef0250708f478b944675dcce078c1ff) * fix e2e test and unit test Signed-off-by: Qiaozp (cherry picked from commit 0f4e030b84e9bb08697277cafb1116fcf499af72) * add context e2e test Signed-off-by: Qiaozp (cherry picked from commit 88879e6e43f986cb59d4430903305230d30f27d5) * goimports Signed-off-by: Qiaozp (cherry picked from commit 58429978b3c79983ec5baf3851c2bcac9595a4b2) * add more test Signed-off-by: Qiaozp (cherry picked from commit 75760521a460d881d206739fb1c33ccd5884f50c) * review Signed-off-by: Qiaozp (cherry picked from commit cc4c7064667d9a7d5cbe3d36a0946cb0413edc77) * remove optional tag in returned value, unify the imports name Signed-off-by: Qiaozp (cherry picked from commit 658b184aef656cf4d611fddf09028b5a3e45d070) * fix e2e test Signed-off-by: Qiaozp (cherry picked from commit a9e9c96856892b7394f77eeb6cb19fe7cb3552e3) * add stop test Signed-off-by: Qiaozp (cherry picked from commit 58aa2e5125b44fb26a3fe25d36d5a1f1859df480) * more coverage Signed-off-by: Qiaozp (cherry picked from commit 01ecb513233ac6015e09182abef00f92e2a491fe) * single case selct Signed-off-by: Qiaozp (cherry picked from commit d9e8fd03426c811d741dff5567642f5d42f0ad3f) * optimize log color Signed-off-by: Qiaozp (cherry picked from commit be9840c3cb0d4893708a40b778a500d90f941629) * add default permission and role Signed-off-by: Qiaozp (cherry picked from commit cf074444ac51f7dd13131360f3f8dd695307927e) * fix permission ut Signed-off-by: Qiaozp (cherry picked from commit 1bdcad63a2128f6ace72825182d600451ef8cf99) * change the log api implementation Signed-off-by: Qiaozp (cherry picked from commit 587f745430d4f6d6655b7dc84f71a255a147ff49) * add color, add container order Signed-off-by: Qiaozp (cherry picked from commit 6e7f18760585bb5f9d07551ef0e00d8fc21c03b6) * lint Signed-off-by: Qiaozp (cherry picked from commit 64ba029031c6d8b1e13dea16544d561740e8f976) * fix filter nil will cut all log Signed-off-by: Qiaozp (cherry picked from commit 799dfe377ad8ba3b4868c93ea5b98427c2be5fed) * longer timeout and lint Signed-off-by: Qiaozp (cherry picked from commit 89873f1f66d167507da710c394b79f3f542b60ff) Co-authored-by: Qiaozp Co-authored-by: barnettZQG --- .github/workflows/apiserver-test.yaml | 2 + docs/apidoc/swagger.json | 2031 +++++++++-------- e2e/addon/mock/testrepo/helm-repo/index.yaml | 11 + .../helm-repo/vela-workflow-v0.3.1.tgz | Bin 0 -> 2006 bytes e2e/addon/mock/vela_addon_mock_server.go | 15 +- go.mod | 2 +- pkg/addon/addon.go | 2 +- .../domain/model/{context.go => pipeline.go} | 44 +- pkg/apiserver/domain/model/project.go | 8 +- .../domain/service/cloudshell_test.go | 3 +- pkg/apiserver/domain/service/helm_test.go | 4 +- pkg/apiserver/domain/service/pipeline.go | 1105 +++++++-- pkg/apiserver/domain/service/pipeline_test.go | 143 ++ pkg/apiserver/domain/service/project.go | 15 + pkg/apiserver/domain/service/rbac.go | 24 +- pkg/apiserver/domain/service/rbac_test.go | 2 +- pkg/apiserver/domain/service/suite_test.go | 6 +- pkg/apiserver/domain/service/workflow_test.go | 2 +- .../infrastructure/clients/kubeclient.go | 12 +- pkg/apiserver/interfaces/api/dto/v1/types.go | 83 +- pkg/apiserver/interfaces/api/interfaces.go | 2 +- pkg/apiserver/interfaces/api/pipeline.go | 336 ++- pkg/apiserver/interfaces/api/project.go | 13 +- pkg/apiserver/utils/bcode/017_pipeline.go | 18 + pkg/utils/k8s.go | 134 ++ references/cli/logs.go | 120 +- test/e2e-apiserver-test/pipeline_test.go | 363 +++ test/e2e-apiserver-test/suite_test.go | 3 +- test/e2e-apiserver-test/testdata/log.yaml | 35 + test/e2e-apiserver-test/testdata/request.yaml | 43 + 30 files changed, 3154 insertions(+), 1427 deletions(-) create mode 100644 e2e/addon/mock/testrepo/helm-repo/vela-workflow-v0.3.1.tgz rename pkg/apiserver/domain/model/{context.go => pipeline.go} (62%) create mode 100644 pkg/apiserver/domain/service/pipeline_test.go create mode 100644 test/e2e-apiserver-test/pipeline_test.go create mode 100644 test/e2e-apiserver-test/testdata/log.yaml create mode 100644 test/e2e-apiserver-test/testdata/request.yaml diff --git a/.github/workflows/apiserver-test.yaml b/.github/workflows/apiserver-test.yaml index 8a1661cda..b44a0eb18 100644 --- a/.github/workflows/apiserver-test.yaml +++ b/.github/workflows/apiserver-test.yaml @@ -176,10 +176,12 @@ jobs: make e2e-cleanup make e2e-setup-core bin/vela addon enable fluxcd + bin/vela addon enable vela-workflow timeout 600s bash -c -- 'while true; do kubectl get ns flux-system; if [ $? -eq 0 ] ; then break; else sleep 5; fi;done' kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-core,app.kubernetes.io/instance=kubevela -n vela-system --timeout=600s kubectl wait --for=condition=Ready pod -l app=source-controller -n flux-system --timeout=600s kubectl wait --for=condition=Ready pod -l app=helm-controller -n flux-system --timeout=600s + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-workflow -n vela-system --timeout=600s - name: Run api server e2e test run: | diff --git a/docs/apidoc/swagger.json b/docs/apidoc/swagger.json index 624f824ce..9961a4213 100644 --- a/docs/apidoc/swagger.json +++ b/docs/apidoc/swagger.json @@ -4200,900 +4200,6 @@ } } }, - "/api/v1/pipelines": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "list pipelines", - "operationId": "listPipelines", - "parameters": [ - { - "type": "string", - "description": "Fuzzy search based on name or description", - "name": "query", - "in": "query" - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.ListPipelineResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "post": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "create pipeline", - "operationId": "createPipeline", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.CreatePipelineRequest" - } - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineBase" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "get pipeline", - "operationId": "getPipeline", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.GetPipelineRequest" - } - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.GetPipelineResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "put": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "update pipeline", - "operationId": "updatePipeline", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.UpdatePipelineRequest" - } - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineBase" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "delete": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "delete pipeline", - "operationId": "deletePipeline", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineMetaResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/contexts": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "list pipeline context values", - "operationId": "listContextValues", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.ListContextValueResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "post": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "create pipeline context values", - "operationId": "createContextValue", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.CreateContextValuesRequest" - } - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.Context" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/contexts/{contextName}": { - "put": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "update pipeline context value", - "operationId": "updateContextValue", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.UpdateContextValuesRequest" - } - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline context name", - "name": "contextName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.Context" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "delete": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "delete pipeline context value", - "operationId": "deleteContextValue", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline context name", - "name": "contextName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.ContextNameResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/run": { - "post": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "run pipeline", - "operationId": "runPipeline", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.RunPipelineRequest" - } - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineRunMeta" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "list pipeline runs", - "operationId": "listPipelineRuns", - "parameters": [ - { - "type": "string", - "description": "query identifier of the status", - "name": "status", - "in": "query" - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.ListPipelineRunResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs/{runName}": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "get pipeline run", - "operationId": "getPipelineRun", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineRunBase" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "delete": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "delete pipeline run", - "operationId": "deletePipelineRun", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineRunMeta" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs/{runName}/log": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "get pipeline run log", - "operationId": "getPipelineRunLog", - "parameters": [ - { - "type": "string", - "description": "query by specific id", - "name": "step", - "in": "query" - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.GetPipelineRunLogResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs/{runName}/output": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "get pipeline run output", - "operationId": "getPipelineRunOutput", - "parameters": [ - { - "type": "string", - "description": "query by specific id", - "name": "step", - "in": "query" - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.GetPipelineRunOutputResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs/{runName}/status": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "get pipeline run status", - "operationId": "getPipelineRunStatus", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1alpha1.WorkflowRunStatus" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs/{runName}/stop": { - "post": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "stop pipeline run", - "operationId": "stopPipeline", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineRunMeta" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, "/api/v1/projects": { "get": { "consumes": [ @@ -5152,6 +4258,45 @@ } } }, + "/api/v1/projects/pipelines": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "list pipelines", + "operationId": "listPipelines", + "parameters": [ + { + "type": "string", + "description": "Fuzzy search based on name or description", + "name": "query", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.ListPipelineResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, "/api/v1/projects/{projectName}": { "get": { "consumes": [ @@ -5845,6 +4990,863 @@ } } }, + "/api/v1/projects/{projectName}/pipelines": { + "post": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "create pipeline", + "operationId": "createPipeline", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.CreatePipelineRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineBase" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "get pipeline", + "operationId": "getPipeline", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.GetPipelineRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.GetPipelineResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + }, + "put": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "update pipeline", + "operationId": "updatePipeline", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.UpdatePipelineRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineBase" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + }, + "delete": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "delete pipeline", + "operationId": "deletePipeline", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineMetaResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/contexts": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "list pipeline context values", + "operationId": "listContextValues", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.ListContextValueResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + }, + "post": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "create pipeline context values", + "operationId": "createContextValue", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.CreateContextValuesRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.Context" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/contexts/{contextName}": { + "put": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "update pipeline context value", + "operationId": "updateContextValue", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.UpdateContextValuesRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline context name", + "name": "contextName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.Context" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + }, + "delete": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "delete pipeline context value", + "operationId": "deleteContextValue", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline context name", + "name": "contextName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.ContextNameResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/run": { + "post": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "run pipeline", + "operationId": "runPipeline", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.RunPipelineRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineRunMeta" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "list pipeline runs", + "operationId": "listPipelineRuns", + "parameters": [ + { + "type": "string", + "description": "query identifier of the status", + "name": "status", + "in": "query" + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.ListPipelineRunResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs/{runName}": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "get pipeline run", + "operationId": "getPipelineRun", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineRunBase" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + }, + "delete": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "delete pipeline run", + "operationId": "deletePipelineRun", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineRunMeta" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs/{runName}/log": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "get pipeline run log", + "operationId": "getPipelineRunLog", + "parameters": [ + { + "type": "string", + "description": "query by specific step name", + "name": "step", + "in": "query" + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.GetPipelineRunLogResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs/{runName}/output": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "get pipeline run output", + "operationId": "getPipelineRunOutput", + "parameters": [ + { + "type": "string", + "description": "query by specific id", + "name": "step", + "in": "query" + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.GetPipelineRunOutputResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs/{runName}/status": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "get pipeline run status", + "operationId": "getPipelineRunStatus", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1alpha1.WorkflowRunStatus" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs/{runName}/stop": { + "post": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "stop pipeline run", + "operationId": "stopPipeline", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineRunMeta" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, "/api/v1/projects/{projectName}/providers": { "get": { "consumes": [ @@ -7355,6 +7357,12 @@ "in": "path", "required": true }, + { + "type": "string", + "description": "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + "name": "dryRun", + "in": "query" + }, { "name": "body", "in": "body", @@ -8358,8 +8366,8 @@ }, "config.ClusterTargetStatus": { "required": [ - "namespace", "clusterName", + "namespace", "status", "application", "message" @@ -8648,8 +8656,8 @@ }, "model.Cluster": { "required": [ - "createTime", "updateTime", + "createTime", "name", "alias", "description", @@ -9444,12 +9452,12 @@ }, "v1.ApplicationDeployResponse": { "required": [ - "status", "note", - "createTime", - "envName", "triggerType", - "version" + "createTime", + "version", + "status", + "envName" ], "properties": { "codeInfo": { @@ -10209,13 +10217,13 @@ }, "v1.ConfigTemplateDetail": { "required": [ + "alias", + "name", "namespace", "description", "scope", "sensitive", "createTime", - "alias", - "name", "schema", "uiSchema" ], @@ -11035,11 +11043,11 @@ }, "v1.DetailAddonResponse": { "required": [ + "invisible", + "name", "version", "description", "icon", - "name", - "invisible", "schema", "uiSchema", "definitions", @@ -11119,13 +11127,13 @@ }, "v1.DetailApplicationResponse": { "required": [ - "updateTime", - "name", "alias", - "project", "description", - "createTime", "icon", + "name", + "project", + "createTime", + "updateTime", "policies", "envBindings", "resourceInfo" @@ -11182,20 +11190,20 @@ }, "v1.DetailClusterResponse": { "required": [ - "updateTime", - "name", "apiServerURL", - "kubeConfigSecret", - "createTime", "alias", - "provider", - "kubeConfig", "description", "icon", - "reason", + "kubeConfigSecret", + "name", "labels", - "status", + "reason", + "provider", "dashboardURL", + "updateTime", + "createTime", + "status", + "kubeConfig", "resourceInfo" ], "properties": { @@ -11253,14 +11261,14 @@ }, "v1.DetailComponentResponse": { "required": [ - "appPrimaryKey", + "updateTime", + "creator", + "alias", "main", "createTime", "type", "name", - "alias", - "updateTime", - "creator", + "appPrimaryKey", "definition" ], "properties": { @@ -11348,13 +11356,13 @@ }, "v1.DetailDefinitionResponse": { "required": [ + "alias", "description", "status", - "ownerAddon", - "name", - "alias", - "icon", "labels", + "name", + "icon", + "ownerAddon", "schema", "uiSchema" ], @@ -11411,15 +11419,15 @@ }, "v1.DetailPolicyResponse": { "required": [ - "name", + "envName", "alias", + "description", "creator", "properties", "createTime", - "updateTime", - "envName", + "name", "type", - "description" + "updateTime" ], "properties": { "alias": { @@ -11461,18 +11469,18 @@ }, "v1.DetailRevisionResponse": { "required": [ - "version", + "triggerType", + "note", "revisionCRName", - "workflowName", - "reason", - "updateTime", + "status", "appPrimaryKey", + "version", "envName", "createTime", - "status", + "reason", "deployUser", - "note", - "triggerType" + "workflowName", + "updateTime" ], "properties": { "appPrimaryKey": { @@ -11529,8 +11537,8 @@ }, "v1.DetailTargetResponse": { "required": [ - "updateTime", "project", + "updateTime", "name", "createTime" ], @@ -11572,11 +11580,11 @@ }, "v1.DetailUserResponse": { "required": [ - "email", - "disabled", "createTime", "lastLoginTime", "name", + "email", + "disabled", "projects", "roles" ], @@ -11674,13 +11682,13 @@ }, "v1.DetailWorkflowResponse": { "required": [ + "createTime", + "name", "alias", "enable", - "envName", - "name", - "description", "default", - "createTime", + "description", + "envName", "updateTime" ], "properties": { @@ -11913,8 +11921,8 @@ }, "v1.GetPipelineResponse": { "required": [ - "project", "name", + "project", "spec", "info" ], @@ -11941,14 +11949,27 @@ }, "v1.GetPipelineRunLogResponse": { "required": [ + "phase", + "id", + "name", + "type", "log" ], "properties": { + "id": { + "type": "string" + }, "log": { - "type": "array", - "items": { - "$ref": "#/definitions/v1.Log" - } + "type": "string" + }, + "name": { + "type": "string" + }, + "phase": { + "type": "string" + }, + "type": { + "type": "string" } } }, @@ -11960,7 +11981,7 @@ "output": { "type": "array", "items": { - "$ref": "#/definitions/v1.Output" + "$ref": "#/definitions/v1.StepOutput" } } } @@ -12526,32 +12547,6 @@ } } }, - "v1.Log": { - "required": [ - "id", - "name", - "type", - "phase", - "log" - ], - "properties": { - "id": { - "type": "string" - }, - "log": { - "type": "string" - }, - "name": { - "type": "string" - }, - "phase": { - "type": "string" - }, - "type": { - "type": "string" - } - } - }, "v1.LoginRequest": { "properties": { "code": { @@ -12729,35 +12724,6 @@ } } }, - "v1.Output": { - "required": [ - "id", - "name", - "type", - "phase", - "vars" - ], - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "phase": { - "type": "string" - }, - "type": { - "type": "string" - }, - "vars": { - "type": "object", - "additionalProperties": { - "type": "string" - } - } - } - }, "v1.PermissionBase": { "required": [ "name", @@ -12802,8 +12768,8 @@ }, "v1.PipelineBase": { "required": [ - "project", "name", + "project", "spec" ], "properties": { @@ -12891,8 +12857,8 @@ }, "v1.PipelineMetaResponse": { "required": [ - "name", - "project" + "project", + "name" ], "properties": { "alias": { @@ -12911,9 +12877,9 @@ }, "v1.PipelineRunBase": { "required": [ - "project", "pipelineRunName", "pipelineName", + "project", "record", "contextName", "spec" @@ -13331,6 +13297,52 @@ } } }, + "v1.StepOutput": { + "required": [ + "outputs", + "subStepOutputs" + ], + "properties": { + "outputs": { + "$ref": "#/definitions/v1.StepOutputBase" + }, + "subStepOutputs": { + "type": "array", + "items": { + "$ref": "#/definitions/v1.StepOutputBase" + } + } + } + }, + "v1.StepOutputBase": { + "required": [ + "phase", + "id", + "name", + "type", + "vars" + ], + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "phase": { + "type": "string" + }, + "type": { + "type": "string" + }, + "vars": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, "v1.SystemInfo": { "required": [ "platformID", @@ -13390,9 +13402,9 @@ }, "v1.SystemInfoResponse": { "required": [ + "enableCollection", "loginType", "platformID", - "enableCollection", "systemVersion" ], "properties": { @@ -14022,6 +14034,9 @@ }, "v1alpha1.WorkflowRunSpec": { "properties": { + "context": { + "type": "string" + }, "mode": { "$ref": "#/definitions/v1alpha1.WorkflowExecuteMode" }, diff --git a/e2e/addon/mock/testrepo/helm-repo/index.yaml b/e2e/addon/mock/testrepo/helm-repo/index.yaml index e496f9609..8d5bc3211 100644 --- a/e2e/addon/mock/testrepo/helm-repo/index.yaml +++ b/e2e/addon/mock/testrepo/helm-repo/index.yaml @@ -23,4 +23,15 @@ entries: annotations: system.vela: ">=1.5.0" system.kubernetes: ">=1.30.0" + vela-workflow: + - annotations: + system.vela: '>=v1.6.0-beta.1' + created: "2022-10-29T09:11:16.865230605Z" + description: vela-workflow provides the capability to run a standalone workflow + home: https://github.com/kubevela/workflow + icon: https://static.kubevela.net/images/logos/KubeVela%20-03.png + name: vela-workflow + urls: + - http://127.0.0.1:9098/helm/vela-workflow-v0.3.1.tgz + version: v0.3.1 generated: "2022-06-15T13:17:04.733573+08:00" \ No newline at end of file diff --git a/e2e/addon/mock/testrepo/helm-repo/vela-workflow-v0.3.1.tgz b/e2e/addon/mock/testrepo/helm-repo/vela-workflow-v0.3.1.tgz new file mode 100644 index 0000000000000000000000000000000000000000..94495cb851f3e9c78c5cd24a9daa3ec3289a6809 GIT binary patch literal 2006 zcmV;{2Pya;iwFP!000001MOOCZ`(E(&R6qSa0*tmTP@4i0LGiX=z1Btb=j~K?S~?0 zX^FDgNTMcEPL0n0eTSqjmSQ_gmZsRs=YuU#Vi5Z@&*)IAYi|1Hm z{;^YuD2ZggTMFhTRQ6a*=2Y}%B!fHIHMHFi>D<8=ERc(1XW)4b?mg1BWeL!TB81+* ze=imkuJjH;4xV6|kXb}qZUqzd$~@V-Heh1M;x8R1|5dc}e?UW)Fdd$^)(c#}{*RBm zzODavfqnMZG1=uQ&4?Os{hZ#2#})N|G#a+-KY#|n*8e*Ji&LISd>?fjUG8}5pa%#! zE!#U&{0VjBDuvqWl+Mx>lEU;3)HN>kG@++;oZ5?APGUEp%O3qmVyNrgJd1!>Rg5xn z;hM6mik7PQG?C2;>=P&$eBO%*@XJan>Oj zP_-XNW;r!;puzDomMVE+{Fy0W1pf>YysnCqgVsn=U=3}CB8NI3_-u-K>K;B$(`be7 zX&y@w<1CYqWq)v{eNH&9JQu=v_kP^Y6?Vg`u{c)TrnRLn|`Nl6A55G<}X>PhJ| zg{40f{-&Wjzv;#jJ%H-x;U>9kF7WX&e)s3QdW%Y`I3~)W-)e7j9%7R};G$kz<&uZn zEUrpHGpY)Q{EbJ4E+RZ2cpr}cTU+`bj?0^MmI%YL#6-H zec4@MU;pn%lq9NJh#R;58dvoH18;oP>i@mbz_b1TU0_%LU)%l~t^bYMZOy5Ad1ziM zLO;zQO=|X++U{1)G#rbn2WjmSq_rZUYtqnN3zOBK!X!!WtWR%u;+xO(&_?;`geaQPqs^p?kcK?!IF{(Q0fzO>7ijZL8mSa4lPHvr@_f zYo*U)rLU(fel=*<|Jt+uQOR%Miu&&j+V#Ib@y53P-w7aDokK&7W5N=IApV8anWApB zUu`>qm=f*l%8!cFTD&uC_>;4ZlFpD z!#O3Pi7elHZuWHbsboQ;;w5)+Ua1GtOucN7jMkk{MpOPu&nx#(Pw(VeRE-x8i=Qm~ z1<9mn{(i1sA((~w@#a#eoe>q5rEaTPo;xg_fkV!zepj(eH%5B>FNu>%wcGfZ{!P8R zJlo#Dwd23EoBjpmmVE50_#gKtm+Jp;WaIxXU?Qmbe~ZdDm*A6bA2rYG%YXa_AqO#E zllmL~NQwBIi(dIl+_wX->kZvBnL9`asPlLJy5k@u$y`hk#$Eht7&4!c2qBg%nP9UB z_8$^BLm}8`N|S&lzRD7Mwoy+Q 0 { + logs, err = getResourceLogs(ctx, p.KubeConfig, p.KubeClient, logConfig.Source.Resources, nil) + if err != nil { + return apis.GetPipelineRunLogResponse{}, err + } + } + if logConfig.Source.URL != "" { + var logsBuilder strings.Builder + readCloser, err := wfUtils.GetLogsFromURL(ctx, logConfig.Source.URL) + if err != nil { + log.Logger.Errorf("get logs from url %s failed: %v", logConfig.Source.URL, err) + return apis.GetPipelineRunLogResponse{}, bcode.ErrReadSourceLog + } + //nolint:errcheck + defer readCloser.Close() + if _, err := io.Copy(&logsBuilder, readCloser); err != nil { + log.Logger.Errorf("copy logs from url %s failed: %v", logConfig.Source.URL, err) + return apis.GetPipelineRunLogResponse{}, bcode.ErrReadSourceLog + } + logs = logsBuilder.String() + } + } + return apis.GetPipelineRunLogResponse{ + StepBase: getStepBase(pipelineRun, step), + Log: logs, + }, nil +} + +func getStepBase(run apis.PipelineRun, step string) apis.StepBase { + for _, s := range run.Status.Steps { + if s.Name == step { + return apis.StepBase{ + ID: s.ID, + Name: s.Name, + Type: s.Type, + Phase: string(s.Phase), + } + } + } + return apis.StepBase{} +} + +func getStepOutputs(step v1alpha1.StepStatus, outputsSpec map[string]v1alpha1.StepOutputs, v *value.Value) apis.StepOutputBase { + o := apis.StepOutputBase{ + StepBase: apis.StepBase{ + Name: step.Name, + ID: step.ID, + Phase: string(step.Phase), + Type: step.Type, + }, + } + values := make([]apis.OutputVar, 0) + for _, output := range outputsSpec[step.Name] { + outputValue, err := v.LookupValue(output.Name) + if err != nil { + continue + } + s, err := outputValue.String() + if err != nil { + continue + } + values = append(values, apis.OutputVar{ + Name: output.Name, + ValueFrom: output.ValueFrom, + Value: s, + }) + } + o.Values = values + return o +} + +func getStepInputs(step v1alpha1.StepStatus, inputsSpec map[string]v1alpha1.StepInputs, v *value.Value, valueFromStep map[string]string) apis.StepInputBase { + o := apis.StepInputBase{ + StepBase: apis.StepBase{ + Name: step.Name, + ID: step.ID, + Phase: string(step.Phase), + Type: step.Type, + }, + } + values := make([]apis.InputVar, 0) + for _, input := range inputsSpec[step.Name] { + outputValue, err := v.LookupValue(input.From) + if err != nil { + continue + } + s, err := outputValue.String() + if err != nil { + continue + } + values = append(values, apis.InputVar{ + Value: s, + From: input.From, + FromStep: valueFromStep[input.From], + ParameterKey: input.ParameterKey, + }) + } + o.Values = values + return o +} + +func getResourceLogs(ctx context.Context, config *rest.Config, cli client.Client, resources []wfTypes.Resource, filters []string) (string, error) { + var ( + linesOfLogKept int64 = 1000 + timeout = 10 * time.Second + errPrint = color.New(color.FgRed, color.Bold).FprintfFunc() + ) + + type PodContainer struct { + Name string + Namespace string + Container string + Label map[string]string + } + pods, err := wfUtils.GetPodListFromResources(ctx, cli, resources) + if err != nil { + log.Logger.Errorf("fail to get pod list from resources: %v", err) + return "", err + } + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + log.Logger.Errorf("fail to get clientset from kubeconfig: %v", err) + return "", err + } + podContainers := make([]PodContainer, 0) + for _, pod := range pods { + for _, container := range pod.Spec.Containers { + pc := PodContainer{ + Name: pod.Name, + Namespace: pod.Namespace, + Container: container.Name, + Label: pod.Labels, + } + podContainers = append(podContainers, pc) + } + } + + wg := sync.WaitGroup{} + logBuilder := strings.Builder{} + logMap := concurrent.NewMap() + wg.Add(len(podContainers)) + + for _, pc := range podContainers { + go func(pc PodContainer) { + defer wg.Done() + ctxQuery, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + podLogOpts := corev1.PodLogOptions{ + Container: pc.Container, + Follow: false, + TailLines: &linesOfLogKept, + } + req := clientSet.CoreV1().Pods(pc.Namespace).GetLogs(pc.Name, &podLogOpts) + podLogs, err := req.Stream(ctxQuery) + if err != nil { + log.Logger.Errorf("fail to get pod logs: %v", err) + return + } + defer func() { + _ = podLogs.Close() + }() + podColor, containerColor := determineColor(pc.Name) + buf := new(bytes.Buffer) + p := podColor.SprintfFunc() + c := containerColor.SprintfFunc() + fmt.Fprintf(buf, "%s %s\n", p("› %s", pc.Name), c("%s", pc.Container)) + var readErr error + r := bufio.NewReader(podLogs) + for { + s, err := r.ReadString('\n') + if err != nil { + if !errors.Is(err, io.EOF) { + readErr = err + } + break + } + if len(filters) != 0 { + for _, f := range filters { + if strings.Contains(s, f) { + buf.WriteString(s) + } + } + } + } + if readErr != nil { + errPrint(buf, "error in copy information from APIServer to buffer: %s", err.Error()) + log.Logger.Errorf("fail to copy pod logs: %v", err) + } + logMap.Store(fmt.Sprintf("%s/%s", pc.Name, pc.Container), buf.String()) + }(pc) + } + wg.Wait() + + order := make([]string, 0) + sort.Strings(order) + logMap.Range(func(key, value any) bool { + order = append(order, key.(string)) + return true + }) + for _, key := range order { + val, ok := logMap.Load(key) + if ok { + logBuilder.WriteString(fmt.Sprintf("%s", val)) + } + } + return logBuilder.String(), nil +} + // RunPipeline will run a pipeline -func (p pipelineServiceImpl) RunPipeline(ctx context.Context, pipeline apis.PipelineBase, req apis.RunPipelineRequest) error { - // todo get context and set to workflowRun +func (p pipelineServiceImpl) RunPipeline(ctx context.Context, pipeline apis.PipelineBase, req apis.RunPipelineRequest) (*apis.PipelineRun, error) { + if err := checkRunMode(&req.Mode); err != nil { + return nil, err + } + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) run := v1alpha1.WorkflowRun{} version := utils.GenerateVersion("") name := fmt.Sprintf("%s-%s", pipeline.Name, version) run.Name = name - run.Namespace = fmt.Sprintf("%s-project", pipeline.Project) - run.Spec.WorkflowRef = pipeline.Name + run.Namespace = project.GetNamespace() + run.Spec.WorkflowSpec = &pipeline.Spec run.Spec.Mode = &req.Mode - return p.KubeClient.Create(ctx, &run) + run.SetLabels(map[string]string{ + labelPipeline: pipeline.Name, + model.LabelSourceOfTruth: model.FromUX, + }) + + // process the context + if req.ContextName != "" { + ppContext, err := p.ContextService.GetContext(ctx, pipeline.Project.Name, pipeline.Name, req.ContextName) + if err != nil { + return nil, err + } + contextData := make(map[string]interface{}) + for _, pair := range ppContext.Values { + contextData[pair.Key] = pair.Value + } + run.Labels[labelContext] = req.ContextName + run.Spec.Context = util.Object2RawExtension(contextData) + } + + if err := p.KubeClient.Create(ctx, &run); err != nil { + return nil, err + } + + return p.PipelineRunService.GetPipelineRun(ctx, apis.PipelineRunMeta{ + PipelineName: pipeline.Name, + Project: apis.NameAlias{Name: project.Name}, + PipelineRunName: name, + }) +} + +// getPipelineInfo returns the pipeline statistic info +// return error can be nil if pipeline hasn't been run +func (p pipelineServiceImpl) getPipelineInfo(ctx context.Context, wf *model.Pipeline, namespace string) (*apis.PipelineInfo, error) { + var runs v1alpha1.WorkflowRunList + err := p.KubeClient.List(context.Background(), &runs, client.InNamespace(namespace), client.MatchingLabels{labelPipeline: wf.Name}) + if err != nil { + if meta.IsNoMatchError(err) { + weekStat := make([]apis.RunStatInfo, 0) + for i := 0; i < 7; i++ { + weekStat = append(weekStat, apis.RunStatInfo{}) + } + return &apis.PipelineInfo{ + LastRun: nil, + RunStat: apis.RunStat{ + ActiveNum: 0, + Total: apis.RunStatInfo{}, + Week: weekStat, + }, + }, nil + } + return nil, err + } + run := getLastRun(runs.Items) + runStat := getRunStat(runs.Items) + pi := &apis.PipelineInfo{ + RunStat: runStat, + } + if run != nil { + projectName := wf.Project + project, err := p.ProjectService.GetProject(ctx, projectName) + if err != nil { + return nil, err + } + pi.LastRun, err = workflowRun2PipelineRun(*run, project, p.ContextService) + if err != nil { + return nil, err + } + } + return pi, nil +} + +func getRunStat(runs []v1alpha1.WorkflowRun) apis.RunStat { + today := time.Now().Unix() + isActive := func(run v1alpha1.WorkflowRun) bool { + return !run.Status.Finished && !run.Status.Terminated && run.Status.Suspend + } + isSuccess := func(run v1alpha1.WorkflowRun) bool { + return run.Status.Phase == v1alpha1.WorkflowStateSucceeded + } + // returned int x means the (x+1)/7 days of the week, valid number is 0-6 + inThisWeek := func(run v1alpha1.WorkflowRun) (int, bool) { + startTime := run.Status.StartTime.Time.Unix() + // one week, note this is not week of natual, but week of unix timestamp + if today-startTime < 604800 { + return 6 - int((today-startTime)/86400), true + } + return -1, false + } + var ( + act int + success int + fail int + week = make([]apis.RunStatInfo, 7) + ) + + for _, run := range runs { + // total = success + fail + active + day, inWeek := inThisWeek(run) + if inWeek { + week[day].Total++ + } + if isActive(run) { + act++ + } else { + if isSuccess(run) { + if inWeek { + week[day].Success++ + } + success++ + } else { + fail++ + if inWeek { + week[day].Fail++ + } + } + } + + } + return apis.RunStat{ + ActiveNum: act, + Total: apis.RunStatInfo{ + Total: len(runs), + Success: success, + Fail: fail, + }, + Week: week, + } +} + +func getLastRun(runs []v1alpha1.WorkflowRun) *v1alpha1.WorkflowRun { + if len(runs) == 0 { + return nil + } + last := runs[0] + lastStartTime := last.Status.StartTime.Time + for _, wfr := range runs { + wfr.Status.StartTime.After(lastStartTime) + last = wfr + lastStartTime = wfr.Status.StartTime.Time + } + // We don't need managed fields, save some bandwidth + last.ManagedFields = nil + return &last } // GetPipelineRun will get a pipeline run -func (p pipelineRunServiceImpl) GetPipelineRun(ctx context.Context, meta apis.PipelineRunMeta) (apis.PipelineRun, error) { - namespacedName := client.ObjectKey{Name: meta.PipelineName, Namespace: nsForProj(meta.Project)} +func (p pipelineRunServiceImpl) GetPipelineRun(ctx context.Context, meta apis.PipelineRunMeta) (*apis.PipelineRun, error) { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) + namespacedName := client.ObjectKey{Name: meta.PipelineRunName, Namespace: project.GetNamespace()} run := v1alpha1.WorkflowRun{} if err := p.KubeClient.Get(ctx, namespacedName, &run); err != nil { - return apis.PipelineRun{}, err + return nil, err } - return workflowRun2PipelineRun(run), nil + if run.Labels != nil && run.Labels[labelPipeline] != "" { + pipeline := &model.Pipeline{ + Name: run.Labels[labelPipeline], + Project: project.Name, + } + if err := p.Store.Get(ctx, pipeline); err != nil { + log.Logger.Errorf("failed to load the workflow %s", err.Error()) + if errors.Is(err, datastore.ErrRecordNotExist) { + return nil, bcode.ErrPipelineNotExist + } + return nil, err + } + run.Spec.WorkflowSpec = &pipeline.Spec + + } + return workflowRun2PipelineRun(run, project, p.ContextService) } // ListPipelineRuns will list all pipeline runs func (p pipelineRunServiceImpl) ListPipelineRuns(ctx context.Context, base apis.PipelineBase) (apis.ListPipelineRunResponse, error) { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) wfrs := v1alpha1.WorkflowRunList{} - if err := p.KubeClient.List(ctx, &wfrs, client.InNamespace(nsForProj(base.Project))); err != nil { + if err := p.KubeClient.List(ctx, &wfrs, client.InNamespace(project.GetNamespace()), client.MatchingLabels{labelPipeline: base.Name}); err != nil { return apis.ListPipelineRunResponse{}, err } - res := apis.ListPipelineRunResponse{} - for _, wfr := range wfrs.Items { - if wfr.Spec.WorkflowRef == base.Name { - res.Runs = append(res.Runs, p.workflowRun2runBriefing(ctx, wfr)) - } + res := apis.ListPipelineRunResponse{ + Runs: make([]apis.PipelineRunBriefing, 0), } + for _, wfr := range wfrs.Items { + res.Runs = append(res.Runs, p.workflowRun2runBriefing(ctx, wfr, project)) + } + res.Total = int64(len(res.Runs)) return res, nil } // DeletePipelineRun will delete a pipeline run func (p pipelineRunServiceImpl) DeletePipelineRun(ctx context.Context, meta apis.PipelineRunMeta) error { - namespacedName := client.ObjectKey{Name: meta.PipelineName, Namespace: nsForProj(meta.Project)} - run := v1alpha1.WorkflowRun{} - if err := p.KubeClient.Get(ctx, namespacedName, &run); err != nil { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) + run := v1alpha1.WorkflowRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: meta.PipelineRunName, + Namespace: project.GetNamespace(), + }, + } + err := p.KubeClient.Delete(ctx, &run) + return client.IgnoreNotFound(err) +} + +// CleanPipelineRuns will clean all pipeline runs, it equals to call ListPipelineRuns and multiple DeletePipelineRun +func (p pipelineRunServiceImpl) CleanPipelineRuns(ctx context.Context, base apis.PipelineBase) error { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) + wfrs := v1alpha1.WorkflowRunList{} + if err := p.KubeClient.List(ctx, &wfrs, client.InNamespace(project.GetNamespace()), client.MatchingLabels{labelPipeline: base.Name}); err != nil { return err } - return p.KubeClient.Delete(ctx, &run) + for _, wfr := range wfrs.Items { + if err := p.KubeClient.Delete(ctx, wfr.DeepCopy()); err != nil { + return client.IgnoreNotFound(err) + } + } + return nil +} + +// InitContext will init pipeline context record +func (c contextServiceImpl) InitContext(ctx context.Context, projectName, pipelineName string) (*model.PipelineContext, error) { + modelCtx := model.PipelineContext{ + ProjectName: projectName, + PipelineName: pipelineName, + } + if err := c.Store.Get(ctx, &modelCtx); err == nil { + return nil, bcode.ErrContextAlreadyExist + } + modelCtx.Contexts = make(map[string][]model.Value) + if err := c.Store.Add(ctx, &modelCtx); err != nil { + return nil, err + } + return &modelCtx, nil } // GetContext will get a context @@ -263,29 +962,36 @@ func (c contextServiceImpl) GetContext(ctx context.Context, projectName, pipelin } vals, ok := modelCtx.Contexts[name] if !ok { - return nil, errors.New("context not found") + return nil, bcode.ErrContextNotFound } return &apis.Context{Name: name, Values: vals}, nil } // CreateContext will create a context func (c contextServiceImpl) CreateContext(ctx context.Context, projectName, pipelineName string, context apis.Context) (*model.PipelineContext, error) { - modelCtx := model.PipelineContext{ + modelCtx := &model.PipelineContext{ ProjectName: projectName, PipelineName: pipelineName, } - if err := c.Store.Get(ctx, &modelCtx); err != nil { - return nil, err + if err := c.Store.Get(ctx, modelCtx); err != nil { + if errors.Is(err, datastore.ErrRecordNotExist) { + modelCtx, err = c.InitContext(ctx, projectName, pipelineName) + if err != nil { + return nil, err + } + } else { + return nil, err + } } if _, ok := modelCtx.Contexts[context.Name]; ok { log.Logger.Errorf("context %s already exists", context.Name) return nil, bcode.ErrContextAlreadyExist } modelCtx.Contexts[context.Name] = context.Values - if err := c.Store.Put(ctx, &modelCtx); err != nil { + if err := c.Store.Put(ctx, modelCtx); err != nil { return nil, err } - return &modelCtx, nil + return modelCtx, nil } // UpdateContext will update a context @@ -311,6 +1017,12 @@ func (c contextServiceImpl) ListContexts(ctx context.Context, projectName, pipel PipelineName: pipelineName, } if err := c.Store.Get(ctx, &modelCtx); err != nil { + if errors.Is(err, datastore.ErrRecordNotExist) { + return &apis.ListContextValueResponse{ + Total: 0, + Contexts: make(map[string][]model.Value), + }, nil + } return nil, err } return &apis.ListContextValueResponse{ @@ -329,99 +1041,164 @@ func (c contextServiceImpl) DeleteContext(ctx context.Context, projectName, pipe return err } delete(modelCtx.Contexts, name) - if err := c.Store.Put(ctx, &modelCtx); err != nil { - return err + return c.Store.Put(ctx, &modelCtx) +} + +// DeleteAllContexts will delete all contexts of a pipeline +func (c contextServiceImpl) DeleteAllContexts(ctx context.Context, projectName, pipelineName string) error { + modelCtx := model.PipelineContext{ + ProjectName: projectName, + PipelineName: pipelineName, } - return nil + return c.Store.Delete(ctx, &modelCtx) } -func nsForProj(proj string) string { - return fmt.Sprintf("project-%s", proj) -} - -func getWfDescription(wf v1alpha1.Workflow) string { - if wf.Labels == nil { - return "" - } - return wf.Labels[labelDescription] -} - -func getWfAlias(wf v1alpha1.Workflow) string { - if wf.Labels == nil { - return "" - } - return wf.Labels[labelAlias] -} - -func fuzzyMatch(wf v1alpha1.Workflow, q string) bool { +func fuzzyMatch(wf *model.Pipeline, q string) bool { if strings.Contains(wf.Name, q) { return true } - if strings.Contains(getWfAlias(wf), q) { + if strings.Contains(wf.Alias, q) { return true } - if strings.Contains(getWfDescription(wf), q) { + if strings.Contains(wf.Description, q) { return true } return false } -func workflow2PipelineBase(wf v1alpha1.Workflow) *apis.PipelineBase { - project := strings.TrimRight(wf.Namespace, "-project") +func pipeline2PipelineBase(wf *model.Pipeline, project model.Project) *apis.PipelineBase { return &apis.PipelineBase{ PipelineMeta: apis.PipelineMeta{ - Name: wf.Name, - Project: project, - Description: getWfDescription(wf), - Alias: getWfAlias(wf), + Name: wf.Name, + Project: apis.NameAlias{ + Name: project.Name, + Alias: project.Alias, + }, + Description: wf.Description, + Alias: wf.Alias, + CreateTime: wf.CreateTime, }, - Spec: wf.WorkflowSpec, + Spec: wf.Spec, } } -func workflowRun2PipelineRun(run v1alpha1.WorkflowRun) apis.PipelineRun { - return apis.PipelineRun{ +func workflowRun2PipelineRun(run v1alpha1.WorkflowRun, project *model.Project, ctxService ContextService) (*apis.PipelineRun, error) { + mergeSteps(&run) + pipelineName := run.Labels[labelPipeline] + pipelineRun := &apis.PipelineRun{ PipelineRunBase: apis.PipelineRunBase{ PipelineRunMeta: apis.PipelineRunMeta{ - PipelineName: run.Spec.WorkflowRef, - Project: strings.TrimRight(run.Namespace, "-project"), + PipelineName: pipelineName, + Project: apis.NameAlias{ + Name: project.Name, + Alias: project.Alias, + }, PipelineRunName: run.Name, }, + Spec: run.Spec, }, Status: run.Status, } + if labels := run.GetLabels(); labels != nil { + ctxName := labels[labelContext] + if ctxName != "" { + ctx, err := ctxService.GetContext(context.Background(), project.Name, pipelineName, ctxName) + if err != nil && !errors.Is(err, datastore.ErrRecordNotExist) { + return nil, err + } + pipelineRun.PipelineRunBase.ContextName = ctxName + pipelineRun.PipelineRunBase.ContextValues = ctx.Values + } + } + return pipelineRun, nil } -func (p pipelineRunServiceImpl) workflowRun2runBriefing(ctx context.Context, run v1alpha1.WorkflowRun) apis.PipelineRunBriefing { - contextName := run.Labels[labelContextName] - project := strings.TrimRight(run.Namespace, "-project") - apiContext, err := p.ContextService.GetContext(ctx, project, run.Spec.WorkflowRef, contextName) - if err != nil { - log.Logger.Warnf("failed to get pipeline run context %s/%s/%s: %v", project, run.Spec.WorkflowRef, contextName, err) - apiContext = nil +func mergeSteps(run *v1alpha1.WorkflowRun) { + if run.Spec.WorkflowSpec == nil { + return + } + if run.Status.Steps == nil { + run.Status.Steps = make([]v1alpha1.WorkflowStepStatus, 0) + } + var stepStatus = make(map[string]*v1alpha1.WorkflowStepStatus, len(run.Status.Steps)) + for i, step := range run.Status.Steps { + stepStatus[step.Name] = &run.Status.Steps[i] + } + for _, step := range run.Spec.WorkflowSpec.Steps { + if stepStatusCache, exist := stepStatus[step.Name]; !exist { + var subSteps []v1alpha1.StepStatus + for _, subStep := range step.SubSteps { + subSteps = append(subSteps, v1alpha1.StepStatus{ + Name: subStep.Name, + Type: subStep.Type, + Phase: v1alpha1.WorkflowStepPhasePending, + }) + } + run.Status.Steps = append(run.Status.Steps, v1alpha1.WorkflowStepStatus{ + StepStatus: v1alpha1.StepStatus{ + Name: step.Name, + Type: step.Type, + Phase: v1alpha1.WorkflowStepPhasePending, + }, + SubStepsStatus: subSteps, + }) + } else if len(step.SubSteps) > len(stepStatusCache.SubStepsStatus) { + var subStepStatus = make(map[string]v1alpha1.StepStatus, len(stepStatusCache.SubStepsStatus)) + for i, step := range stepStatusCache.SubStepsStatus { + subStepStatus[step.Name] = stepStatusCache.SubStepsStatus[i] + } + for _, subStep := range step.SubSteps { + if _, exist := subStepStatus[subStep.Name]; !exist { + stepStatusCache.SubStepsStatus = append(stepStatusCache.SubStepsStatus, v1alpha1.StepStatus{ + Name: subStep.Name, + Type: subStep.Type, + Phase: v1alpha1.WorkflowStepPhasePending, + }) + } + } + } + } +} + +func (p pipelineRunServiceImpl) workflowRun2runBriefing(ctx context.Context, run v1alpha1.WorkflowRun, project *model.Project) apis.PipelineRunBriefing { + var ( + apiContext *apis.Context + err error + ) + pipelineName := run.Labels[labelPipeline] + if contextName, ok := run.Labels[labelContext]; ok { + apiContext, err = p.ContextService.GetContext(ctx, project.Name, pipelineName, contextName) + if err != nil { + log.Logger.Warnf("failed to get pipeline run context %s/%s/%s: %v", project, pipelineName, contextName, err) + apiContext = nil + } } - return apis.PipelineRunBriefing{ + briefing := apis.PipelineRunBriefing{ PipelineRunName: run.Name, Finished: run.Status.Finished, Phase: run.Status.Phase, Message: run.Status.Message, StartTime: run.Status.StartTime, EndTime: run.Status.EndTime, - ContextName: apiContext.Name, - ContextValues: apiContext.Values, } + if apiContext != nil { + briefing.ContextName = apiContext.Name + briefing.ContextValues = apiContext.Values + } + return briefing } -func (p pipelineRunServiceImpl) checkRecordRunning(ctx context.Context, pipelineRun apis.PipelineRunBase) (*v1alpha1.WorkflowRun, error) { +func (p pipelineRunServiceImpl) checkRunNotFinished(ctx context.Context, pipelineRun apis.PipelineRunBase) (*v1alpha1.WorkflowRun, error) { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) run := v1alpha1.WorkflowRun{} if err := p.KubeClient.Get(ctx, types.NamespacedName{ - Namespace: nsForProj(pipelineRun.Project), + Namespace: project.GetNamespace(), Name: pipelineRun.PipelineRunName, }, &run); err != nil { return nil, err } - if !run.Status.Suspend && !run.Status.Terminated && !run.Status.Finished { - return nil, fmt.Errorf("workflow is still running, can not operate a running workflow") + if run.Status.Terminated || run.Status.Finished { + return nil, bcode.ErrPipelineRunFinished } return &run, nil } @@ -461,3 +1238,59 @@ func (p pipelineRunServiceImpl) terminatePipelineRun(ctx context.Context, run *v return nil } + +func checkPipelineSpec(spec v1alpha1.WorkflowSpec) error { + if len(spec.Steps) == 0 { + return bcode.ErrNoSteps + } + return nil +} + +func checkRunMode(mode *v1alpha1.WorkflowExecuteMode) error { + if mode.Steps == "" { + mode.Steps = v1alpha1.WorkflowModeStep + } + if mode.SubSteps == "" { + mode.SubSteps = v1alpha1.WorkflowModeDAG + } + if mode.Steps != v1alpha1.WorkflowModeStep && mode.Steps != v1alpha1.WorkflowModeDAG && + mode.SubSteps != v1alpha1.WorkflowModeStep && mode.SubSteps != v1alpha1.WorkflowModeDAG { + return bcode.ErrWrongMode + } + return nil +} + +// NewTestPipelineService create the pipeline service instance for testing +func NewTestPipelineService(ds datastore.DataStore, c client.Client, cfg *rest.Config) PipelineService { + projectService := NewTestProjectService(ds, c) + contextService := NewTestContextService(ds) + ppRunService := NewTestPipelineRunService(ds, c, cfg) + pipelineService := &pipelineServiceImpl{ + ProjectService: projectService, + ContextService: contextService, + KubeClient: c, + KubeConfig: cfg, + PipelineRunService: ppRunService, + Store: ds, + } + return pipelineService +} + +// NewTestPipelineRunService create the pipeline run service instance for testing +func NewTestPipelineRunService(ds datastore.DataStore, c client.Client, cfg *rest.Config) PipelineRunService { + contextService := NewTestContextService(ds) + projectService := NewTestProjectService(ds, c) + return &pipelineRunServiceImpl{ + KubeClient: c, + KubeConfig: cfg, + ContextService: contextService, + ProjectService: projectService, + } +} + +// NewTestContextService create the context service instance for testing +func NewTestContextService(ds datastore.DataStore) ContextService { + return &contextServiceImpl{ + Store: ds, + } +} diff --git a/pkg/apiserver/domain/service/pipeline_test.go b/pkg/apiserver/domain/service/pipeline_test.go new file mode 100644 index 000000000..5214a6254 --- /dev/null +++ b/pkg/apiserver/domain/service/pipeline_test.go @@ -0,0 +1,143 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + + "github.com/kubevela/workflow/api/v1alpha1" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/oam-dev/kubevela/pkg/apiserver/domain/model" + "github.com/oam-dev/kubevela/pkg/apiserver/infrastructure/datastore" + apisv1 "github.com/oam-dev/kubevela/pkg/apiserver/interfaces/api/dto/v1" + "github.com/oam-dev/kubevela/pkg/oam/util" +) + +var ( + // defaultNamespace = "project-default-ns1-test" + pipelineService *pipelineServiceImpl + pipelineRunService *pipelineRunServiceImpl + userService *userServiceImpl + contextService *contextServiceImpl + projectService *projectServiceImpl + ctx context.Context + + pipelineName = "test-pipeline" + projectName = "test-project" +) +var _ = Describe("Test pipeline service functions", func() { + It("Init services and project", func() { + ds, err := NewDatastore(datastore.Config{Type: "kubeapi", Database: "pipeline-test-kubevela"}) + Expect(ds).ToNot(BeNil()) + Expect(err).Should(BeNil()) + Expect(err).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + pipelineService = NewTestPipelineService(ds, k8sClient, cfg).(*pipelineServiceImpl) + pipelineRunService = pipelineService.PipelineRunService.(*pipelineRunServiceImpl) + contextService = pipelineService.ContextService.(*contextServiceImpl) + projectService = pipelineService.ProjectService.(*projectServiceImpl) + userService = &userServiceImpl{Store: ds, K8sClient: k8sClient} + + ctx = context.WithValue(context.TODO(), &apisv1.CtxKeyUser, "admin") + err = userService.Init(context.TODO()) + Expect(err).Should(BeNil()) + _, err = projectService.CreateProject(ctx, apisv1.CreateProjectRequest{ + Name: projectName, + Owner: "admin", + }) + Expect(err).Should(BeNil()) + projModel, err := projectService.GetProject(context.TODO(), projectName) + Expect(err).Should(BeNil()) + ctx = context.WithValue(ctx, &apisv1.CtxKeyProject, projModel) + }) + + It("Test create pipeline", func() { + rawProps := []byte(`{"url":"https://api.github.com/repos/kubevela/kubevela"}`) + testPipelineSteps := []v1alpha1.WorkflowStep{ + { + SubSteps: []v1alpha1.WorkflowStepBase{ + { + Name: "request", + Type: "request", + Outputs: v1alpha1.StepOutputs{ + { + ValueFrom: "import \"strconv\"\n\"Current star count: \" + strconv.FormatInt(response[\"stargazers_count\"], 10)\n", + Name: "stars", + }, + }, + Properties: &runtime.RawExtension{ + Raw: rawProps, + }, + }, + }, + WorkflowStepBase: v1alpha1.WorkflowStepBase{ + Name: "step-group", + Type: "step-group", + }, + }, + } + + By("create pipeline with sub-steps") + pipeline, err := pipelineService.CreatePipeline(ctx, apisv1.CreatePipelineRequest{ + Name: pipelineName, + Spec: v1alpha1.WorkflowSpec{ + Steps: testPipelineSteps, + }, + }) + Expect(err).Should(BeNil()) + Expect(pipeline.Name).Should(Equal(pipelineName)) + Expect(pipeline.Spec.Steps[0].Name).Should(Equal("step-group")) + }) + + It("list pipeline", func() { + pipelines, err := pipelineService.ListPipelines(ctx, apisv1.ListPipelineRequest{ + Detailed: true, + }) + Expect(err).Should(BeNil()) + Expect(pipelines).ShouldNot(BeNil()) + Expect(pipelines.Total).Should(Equal(1)) + Expect(len(pipelines.Pipelines)).Should(Equal(1)) + Expect(pipelines.Pipelines[0].Info).ShouldNot(BeNil()) + }) + + It("get pipeline contexts", func() { + By("no context") + contexts, err := contextService.ListContexts(ctx, projectName, pipelineName) + Expect(err).Should(BeNil()) + Expect(contexts.Total).Should(Equal(0)) + Expect(len(contexts.Contexts)).Should(Equal(0)) + + By("create context") + contextName := "test-context" + contextKey := "test-key" + contextVal := "test-val" + ppCtx := apisv1.Context{ + Name: contextName, + Values: []model.Value{ + { + Key: contextKey, + Value: contextVal, + }, + }, + } + context, err := contextService.CreateContext(ctx, projectName, pipelineName, ppCtx) + Expect(err).Should(BeNil()) + Expect(len(context.Contexts)).Should(Equal(1)) + }) +}) diff --git a/pkg/apiserver/domain/service/project.go b/pkg/apiserver/domain/service/project.go index 73227e896..94fe75db8 100644 --- a/pkg/apiserver/domain/service/project.go +++ b/pkg/apiserver/domain/service/project.go @@ -22,6 +22,7 @@ import ( "fmt" terraformapi "github.com/oam-dev/terraform-controller/api/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/oam-dev/kubevela/apis/types" @@ -31,6 +32,7 @@ import ( "github.com/oam-dev/kubevela/pkg/apiserver/utils/bcode" "github.com/oam-dev/kubevela/pkg/apiserver/utils/log" "github.com/oam-dev/kubevela/pkg/multicluster" + "github.com/oam-dev/kubevela/pkg/utils" ) // ProjectService project manage service. @@ -159,6 +161,13 @@ func (p *projectServiceImpl) GetProject(ctx context.Context, projectName string) } return nil, err } + if _, err := utils.GetNamespace(ctx, p.K8sClient, project.GetNamespace()); err != nil { + if apierrors.IsNotFound(err) { + if err := utils.CreateNamespace(ctx, p.K8sClient, projectName); err != nil && !apierrors.IsAlreadyExists(err) { + return nil, bcode.ErrProjectNamespaceFail + } + } + } return project, nil } @@ -319,11 +328,16 @@ func (p *projectServiceImpl) CreateProject(ctx context.Context, req apisv1.Creat } } + if err := utils.CreateNamespace(ctx, p.K8sClient, req.Name); err != nil && !apierrors.IsAlreadyExists(err) { + return nil, bcode.ErrProjectNamespaceFail + } + newProject := &model.Project{ Name: req.Name, Description: req.Description, Alias: req.Alias, Owner: owner, + Namespace: req.Name, } if err := p.Store.Add(ctx, newProject); err != nil { @@ -526,6 +540,7 @@ func ConvertProjectModel2Base(project *model.Project, owner *model.User) *apisv1 CreateTime: project.CreateTime, UpdateTime: project.UpdateTime, Owner: apisv1.NameAlias{Name: project.Owner}, + Namespace: project.GetNamespace(), } if owner != nil && owner.Name == project.Owner { base.Owner = apisv1.NameAlias{Name: owner.Name, Alias: owner.Alias} diff --git a/pkg/apiserver/domain/service/rbac.go b/pkg/apiserver/domain/service/rbac.go index eb626f3e3..22c909773 100644 --- a/pkg/apiserver/domain/service/rbac.go +++ b/pkg/apiserver/domain/service/rbac.go @@ -59,6 +59,7 @@ var defaultProjectPermissionTemplate = []*model.PermissionTemplate{ "project:{projectName}/permission:*", "project:{projectName}/environment:*", "project:{projectName}/application:*/*", + "project:{projectName}/pipeline:*/*", }, Actions: []string{"detail", "list"}, Effect: "Allow", @@ -96,6 +97,16 @@ var defaultProjectPermissionTemplate = []*model.PermissionTemplate{ Effect: "Allow", Scope: "project", }, + { + Name: "pipeline-management", + Alias: "Pipeline Management", + Resources: []string{ + "project:{projectName}/pipeline:*", + }, + Actions: []string{"*"}, + Effect: "Allow", + Scope: "project", + }, } var defaultPlatformPermission = []*model.PermissionTemplate{ @@ -234,6 +245,17 @@ var ResourceMaps = map[string]resourceMetadata{ pathName: "configName", }, "provider": {}, + "pipeline": { + pathName: "pipelineName", + subResources: map[string]resourceMetadata{ + "context": { + pathName: "contextName", + }, + "pipelineRun": { + pathName: "pipelineRunName", + }, + }, + }, }, pathName: "projectName", }, @@ -866,7 +888,7 @@ func (p *rbacServiceImpl) InitDefaultRoleAndUsersForProject(ctx context.Context, }, &model.Role{ Name: "project-admin", Alias: "Project Admin", - Permissions: []string{"project-view", "app-management", "env-management", "role-management", "configuration-read"}, + Permissions: []string{"project-view", "app-management", "env-management", "role-management", "pipeline-management", "configuration-read"}, Project: project.Name, }, &model.Role{ Name: "project-viewer", diff --git a/pkg/apiserver/domain/service/rbac_test.go b/pkg/apiserver/domain/service/rbac_test.go index 11c442c51..3cc73d0ca 100644 --- a/pkg/apiserver/domain/service/rbac_test.go +++ b/pkg/apiserver/domain/service/rbac_test.go @@ -198,7 +198,7 @@ var _ = Describe("Test rbac service", func() { policies, err := rbacService.ListPermissions(context.TODO(), "init-test") Expect(err).Should(BeNil()) - Expect(len(policies)).Should(BeEquivalentTo(int64(5))) + Expect(len(policies)).Should(BeEquivalentTo(int64(6))) }) It("Test UpdatePermission", func() { diff --git a/pkg/apiserver/domain/service/suite_test.go b/pkg/apiserver/domain/service/suite_test.go index 850a910b8..39bcb808b 100644 --- a/pkg/apiserver/domain/service/suite_test.go +++ b/pkg/apiserver/domain/service/suite_test.go @@ -25,6 +25,7 @@ import ( "testing" "time" + "github.com/kubevela/workflow/api/v1alpha1" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -74,7 +75,10 @@ var _ = BeforeSuite(func(done Done) { By("new kube client") cfg.Timeout = time.Minute * 2 - k8sClient, err = client.New(cfg, client.Options{Scheme: common.Scheme}) + scheme := common.Scheme + err = v1alpha1.AddToScheme(scheme) + Expect(err).ShouldNot(HaveOccurred()) + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) Expect(err).Should(BeNil()) Expect(k8sClient).ToNot(BeNil()) By("new kube client success") diff --git a/pkg/apiserver/domain/service/workflow_test.go b/pkg/apiserver/domain/service/workflow_test.go index 7b3f5f265..cee26d258 100644 --- a/pkg/apiserver/domain/service/workflow_test.go +++ b/pkg/apiserver/domain/service/workflow_test.go @@ -58,7 +58,7 @@ var _ = Describe("Test workflow service functions", func() { Expect(ds).ToNot(BeNil()) Expect(err).Should(BeNil()) rbacService := &rbacServiceImpl{Store: ds} - projectService = &projectServiceImpl{Store: ds, RbacService: rbacService} + projectService = &projectServiceImpl{Store: ds, RbacService: rbacService, K8sClient: k8sClient} envService = &envServiceImpl{Store: ds, KubeClient: k8sClient, ProjectService: projectService} envBinding = &envBindingServiceImpl{ Store: ds, diff --git a/pkg/apiserver/infrastructure/clients/kubeclient.go b/pkg/apiserver/infrastructure/clients/kubeclient.go index 436773a31..a92e03437 100644 --- a/pkg/apiserver/infrastructure/clients/kubeclient.go +++ b/pkg/apiserver/infrastructure/clients/kubeclient.go @@ -20,13 +20,13 @@ import ( "fmt" pkgmulticluster "github.com/kubevela/pkg/multicluster" + "github.com/kubevela/workflow/api/v1alpha1" + "github.com/kubevela/workflow/pkg/cue/packages" "k8s.io/client-go/discovery" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" - "github.com/kubevela/workflow/pkg/cue/packages" - apiConfig "github.com/oam-dev/kubevela/pkg/apiserver/config" "github.com/oam-dev/kubevela/pkg/auth" "github.com/oam-dev/kubevela/pkg/oam/discoverymapper" @@ -73,11 +73,13 @@ func GetKubeClient() (client.Client, error) { if kubeConfig == nil { return nil, fmt.Errorf("please call SetKubeConfig first") } - var err error - kubeClient, err = pkgmulticluster.NewClient(kubeConfig, pkgmulticluster.ClientOptions{ + err := v1alpha1.AddToScheme(common.Scheme) + if err != nil { + return nil, err + } + return pkgmulticluster.NewClient(kubeConfig, pkgmulticluster.ClientOptions{ Options: client.Options{Scheme: common.Scheme}, }) - return kubeClient, err } // GetKubeConfig create/get kube runtime config diff --git a/pkg/apiserver/interfaces/api/dto/v1/types.go b/pkg/apiserver/interfaces/api/dto/v1/types.go index a8a8ad298..0138b4c6d 100644 --- a/pkg/apiserver/interfaces/api/dto/v1/types.go +++ b/pkg/apiserver/interfaces/api/dto/v1/types.go @@ -50,12 +50,14 @@ var ( CtxKeyApplicationComponent = "component" // CtxKeyUser request context key of user CtxKeyUser = "user" + // CtxKeyProject request context key of project + CtxKeyProject = "project" // CtxKeyToken request context key of request token CtxKeyToken = "token" // CtxKeyPipeline request context key of pipeline CtxKeyPipeline = "pipeline" - // CtxKeyPipelineContex request context key of pipeline context - CtxKeyPipelineContex = "pipeline-context" + // CtxKeyPipelineContext request context key of pipeline context + CtxKeyPipelineContext = "pipeline-context" // CtxKeyPipelineRun request context key of pipeline run CtxKeyPipelineRun = "pipeline-run" ) @@ -791,6 +793,7 @@ type ProjectBase struct { CreateTime time.Time `json:"createTime"` UpdateTime time.Time `json:"updateTime"` Owner NameAlias `json:"owner,omitempty"` + Namespace string `json:"-"` } // CreateProjectRequest create project request body @@ -1551,10 +1554,11 @@ type ListConfigDistributionResponse struct { // PipelineMeta is metadata of pipeline type PipelineMeta struct { - Name string `json:"name" validate:"checkname"` - Alias string `json:"alias" validate:"checkalias" optional:"true"` - Project string `json:"project"` - Description string `json:"description" optional:"true"` + Name string `json:"name"` + Alias string `json:"alias"` + Project NameAlias `json:"project"` + Description string `json:"description"` + CreateTime time.Time `json:"createTime"` } // PipelineBase is the base info of pipeline @@ -1580,7 +1584,6 @@ type RunStat struct { // CreatePipelineRequest is the request body of creating pipeline type CreatePipelineRequest struct { Name string `json:"name" validate:"checkname"` - Project string `json:"project"` Alias string `json:"alias" validate:"checkalias" optional:"true"` Description string `json:"description" optional:"true"` Spec workflowv1alpha1.WorkflowSpec `json:"spec"` @@ -1593,8 +1596,9 @@ type PipelineMetaResponse struct { // ListPipelineRequest is the request body of listing pipeline type ListPipelineRequest struct { - Projects []string `json:"projects"` - Query string `json:"query"` + Projects []string `json:"projects" optional:"true"` + Query string `json:"query" optional:"true"` + Detailed bool `json:"detailed" optional:"true"` } // ListPipelineResponse is the response body of listing pipeline @@ -1616,11 +1620,6 @@ type UpdatePipelineRequest struct { Spec workflowv1alpha1.WorkflowSpec `json:"spec" optional:"true"` } -// GetPipelineRequest is the request body of getting pipeline -type GetPipelineRequest struct { - Detailed bool `json:"detailed"` -} - // GetPipelineResponse is the response body of getting pipeline type GetPipelineResponse struct { PipelineBase `json:",inline"` @@ -1629,9 +1628,8 @@ type GetPipelineResponse struct { // PipelineInfo is the info of pipeline type PipelineInfo struct { - RelatedApps []ApplicationBase `json:"relatedApps"` - LastRunStatus workflowv1alpha1.WorkflowRunStatus `json:"lastRunStatus"` - RunStat RunStat `json:"runStat"` + LastRun *PipelineRun `json:"lastRun"` + RunStat RunStat `json:"runStat"` } /***********************/ @@ -1652,9 +1650,9 @@ type PipelineRunBriefing struct { // PipelineRunMeta is the metadata of pipeline run type PipelineRunMeta struct { - PipelineName string `json:"pipelineName"` - Project string `json:"project"` - PipelineRunName string `json:"pipelineRunName"` + PipelineName string `json:"pipelineName"` + Project NameAlias `json:"project"` + PipelineRunName string `json:"pipelineRunName"` } // PipelineRun is the info of pipeline run @@ -1667,14 +1665,16 @@ type PipelineRun struct { type PipelineRunBase struct { PipelineRunMeta `json:",inline"` // Record marks the run of the pipeline - Record int64 `json:"record"` - ContextName string `json:"contextName"` - Spec workflowv1alpha1.WorkflowRunSpec `json:"spec"` + Record int64 `json:"record"` + ContextName string `json:"contextName"` + ContextValues []model.Value `json:"contextValues"` + Spec workflowv1alpha1.WorkflowRunSpec `json:"spec"` } // RunPipelineRequest is the request body of running pipeline type RunPipelineRequest struct { // Mode is the mode of the pipeline run. Available values are: "StepByStep", "DAG" for both `step` and `subStep` + // default: "StepByStep" for `step`, "DAG" for `subStep` Mode workflowv1alpha1.WorkflowExecuteMode `json:"mode" optional:"true"` ContextName string `json:"contextName"` } @@ -1687,12 +1687,18 @@ type ListPipelineRunResponse struct { // GetPipelineRunLogResponse is the response body of getting pipeline run log type GetPipelineRunLogResponse struct { - Log []Log `json:"log"` + StepBase `json:",inline"` + Log string `json:"log"` } // GetPipelineRunOutputResponse is the response body of getting pipeline run output type GetPipelineRunOutputResponse struct { - Output []Output `json:"output"` + StepOutputs []StepOutputBase `json:"outputs"` +} + +// GetPipelineRunInputResponse is the response body of getting pipeline run input +type GetPipelineRunInputResponse struct { + StepInputs []StepInputBase `json:"inputs"` } // StepBase is the base info of step @@ -1703,16 +1709,31 @@ type StepBase struct { Phase string `json:"phase"` } -// Log is the log of step -type Log struct { +// StepOutputBase is the output of step +type StepOutputBase struct { StepBase `json:",inline"` - Log string `json:"log"` + Values []OutputVar `json:"values"` } -// Output is the output of step -type Output struct { +// StepInputBase is the input of step +type StepInputBase struct { StepBase `json:",inline"` - Vars map[string]string `json:"vars"` + Values []InputVar `json:"values"` +} + +// OutputVar is one output var +type OutputVar struct { + Name string `json:"name"` + Value string `json:"value"` + ValueFrom string `json:"valueFrom"` +} + +// InputVar is one input var +type InputVar struct { + From string `json:"from"` + FromStep string `json:"fromStep"` + ParameterKey string `json:"parameterKey"` + Value string `json:"value"` } /*******************/ diff --git a/pkg/apiserver/interfaces/api/interfaces.go b/pkg/apiserver/interfaces/api/interfaces.go index e44cc302c..c66f07adc 100644 --- a/pkg/apiserver/interfaces/api/interfaces.go +++ b/pkg/apiserver/interfaces/api/interfaces.go @@ -62,6 +62,7 @@ func InitAPIBean() []interface{} { RegisterAPIInterface(NewApplicationAPIInterface()) RegisterAPIInterface(NewProjectAPIInterface()) RegisterAPIInterface(NewEnvAPIInterface()) + RegisterAPIInterface(NewPipelineAPIInterface()) // Extension RegisterAPIInterface(NewDefinitionAPIInterface()) @@ -82,7 +83,6 @@ func InitAPIBean() []interface{} { RegisterAPIInterface(NewWebhookAPIInterface()) RegisterAPIInterface(NewRepositoryAPIInterface()) RegisterAPIInterface(NewCloudShellAPIInterface()) - RegisterAPIInterface(NewPipelineAPIInterface()) // Authentication RegisterAPIInterface(NewAuthenticationAPIInterface()) diff --git a/pkg/apiserver/interfaces/api/pipeline.go b/pkg/apiserver/interfaces/api/pipeline.go index 95e04847b..695a19fc4 100644 --- a/pkg/apiserver/interfaces/api/pipeline.go +++ b/pkg/apiserver/interfaces/api/pipeline.go @@ -18,199 +18,242 @@ package api import ( "context" + "strconv" restfulspec "github.com/emicklei/go-restful-openapi/v2" "github.com/emicklei/go-restful/v3" - workflowv1alpha1 "github.com/kubevela/workflow/api/v1alpha1" - - "github.com/oam-dev/kubevela/pkg/apiserver/utils/log" + "github.com/kubevela/workflow/api/v1alpha1" + "github.com/pkg/errors" "github.com/oam-dev/kubevela/pkg/apiserver/domain/service" apis "github.com/oam-dev/kubevela/pkg/apiserver/interfaces/api/dto/v1" "github.com/oam-dev/kubevela/pkg/apiserver/utils/bcode" + "github.com/oam-dev/kubevela/pkg/apiserver/utils/log" ) -type pipelineAPIInterface struct { - PipelineService service.PipelineService `inject:""` - PipelineRunService service.PipelineRunService `inject:""` - ContextService service.ContextService `inject:""` -} - -type pipelinePathParamKey string - const ( // Project is the project name key of query param - Project pipelinePathParamKey = "projectName" + Project string = "projectName" // Pipeline is the pipeline name of query param - Pipeline pipelinePathParamKey = "pipelineName" + Pipeline string = "pipelineName" // PipelineRun is the pipeline run name of query param - PipelineRun pipelinePathParamKey = "runName" + PipelineRun string = "runName" // ContextName is the context name of query param - ContextName pipelinePathParamKey = "contextName" + ContextName string = "contextName" ) -// GetWebServiceRoute is the implementation of pipeline Interface -func (p *pipelineAPIInterface) GetWebServiceRoute() *restful.WebService { - - ws := new(restful.WebService) +func initPipelineRoutes(ws *restful.WebService, n *projectAPIInterface) { tags := []string{"pipeline"} - projParam := func(builder *restful.RouteBuilder) { - builder.Param(ws.QueryParameter(string(Project), "project name").Required(true)) + builder.Param(ws.PathParameter(Project, "project name").Required(true)) + builder.Filter(n.projectCheckFilter) } pipelineParam := func(builder *restful.RouteBuilder) { - builder.Param(ws.PathParameter(string(Pipeline), "pipeline name").Required(true)) - builder.Filter(p.pipelineCheckFilter) + builder.Param(ws.PathParameter(Pipeline, "pipeline name").Required(true)) + builder.Filter(n.pipelineCheckFilter) } ctxParam := func(builder *restful.RouteBuilder) { - builder.Param(ws.PathParameter(string(ContextName), "pipeline context name").Required(true)) - builder.Filter(p.pipelineContextCheckFilter) + builder.Param(ws.PathParameter(ContextName, "pipeline context name").Required(true)) + builder.Filter(n.pipelineContextCheckFilter) } runParam := func(builder *restful.RouteBuilder) { - builder.Param(ws.PathParameter(string(PipelineRun), "pipeline run name").Required(true)) - builder.Filter(p.pipelineRunCheckFilter) + builder.Param(ws.PathParameter(PipelineRun, "pipeline run name").Required(true)) + builder.Filter(n.pipelineRunCheckFilter) } meta := func(builder *restful.RouteBuilder) { builder.Metadata(restfulspec.KeyOpenAPITags, tags) } - ws.Path(versionPrefix+"/pipelines"). - Consumes(restful.MIME_JSON, restful.MIME_XML). - Produces(restful.MIME_JSON, restful.MIME_XML). - Doc("api for pipeline manage") - - ws.Route(ws.POST("").To(p.createPipeline). + ws.Route(ws.POST("/{projectName}/pipelines").To(n.createPipeline). Doc("create pipeline"). Reads(apis.CreatePipelineRequest{}). Returns(200, "OK", apis.PipelineBase{}). Returns(400, "Bad Request", bcode.Bcode{}). - Writes(apis.PipelineBase{}).Do(meta)) + Filter(n.RBACService.CheckPerm("project/pipeline", "create")). + Writes(apis.PipelineBase{}).Do(meta, projParam)) - ws.Route(ws.GET("").To(p.listPipelines). - Doc("list pipelines"). - Param(ws.QueryParameter("query", "Fuzzy search based on name or description").DataType("string")). - Returns(200, "OK", apis.ListPipelineResponse{}). - Returns(400, "Bad Request", bcode.Bcode{}). - Writes(apis.ListPipelineResponse{}).Do(meta, projParam)) - - ws.Route(ws.GET("/{pipelineName}").To(p.getPipeline). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}").To(n.getPipeline). Doc("get pipeline"). - Reads(apis.GetPipelineRequest{}). Returns(200, "OK", apis.GetPipelineResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). - Writes(apis.GetPipelineResponse{}).Do(meta, projParam, pipelineParam)) + // use Param instead of pipelineParam to get pipeline information + Param(ws.PathParameter(Pipeline, "pipeline name").Required(true)). + Filter(n.RBACService.CheckPerm("project/pipeline", "detail")). + Writes(apis.GetPipelineResponse{}).Do(meta, projParam)) - ws.Route(ws.PUT("/{pipelineName}").To(p.updatePipeline). + ws.Route(ws.PUT("/{projectName}/pipelines/{pipelineName}").To(n.updatePipeline). Doc("update pipeline"). Reads(apis.UpdatePipelineRequest{}). Returns(200, "OK", apis.PipelineBase{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline", "update")). Writes(apis.PipelineBase{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.DELETE("/{pipelineName}").To(p.deletePipeline). + ws.Route(ws.DELETE("/{projectName}/pipelines/{pipelineName}").To(n.deletePipeline). Doc("delete pipeline"). Returns(200, "OK", apis.PipelineMetaResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline", "delete")). Writes(apis.PipelineMetaResponse{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.POST("/{pipelineName}/contexts").To(p.createContextValue). + ws.Route(ws.POST("/{projectName}/pipelines/{pipelineName}/contexts").To(n.createContextValue). Doc("create pipeline context values"). Reads(apis.CreateContextValuesRequest{}). Returns(200, "OK", apis.Context{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/context", "create")). Writes(apis.Context{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.GET("/{pipelineName}/contexts").To(p.listContextValues). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/contexts").To(n.listContextValues). Doc("list pipeline context values"). Returns(200, "OK", apis.ListContextValueResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/context", "list")). Writes(apis.ListContextValueResponse{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.PUT("/{pipelineName}/contexts/{contextName}").To(p.updateContextValue). + ws.Route(ws.PUT("/{projectName}/pipelines/{pipelineName}/contexts/{contextName}").To(n.updateContextValue). Doc("update pipeline context value"). Reads(apis.UpdateContextValuesRequest{}). Returns(200, "OK", apis.Context{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/context", "update")). Writes(apis.Context{}).Do(meta, projParam, pipelineParam, ctxParam)) - ws.Route(ws.DELETE("/{pipelineName}/contexts/{contextName}").To(p.deleteContextValue). + ws.Route(ws.DELETE("/{projectName}/pipelines/{pipelineName}/contexts/{contextName}").To(n.deleteContextValue). Doc("delete pipeline context value"). Returns(200, "OK", apis.ContextNameResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/context", "delete")). Writes(apis.ContextNameResponse{}).Do(meta, projParam, pipelineParam, ctxParam)) - ws.Route(ws.POST("/{pipelineName}/run").To(p.runPipeline). + ws.Route(ws.POST("/{projectName}/pipelines/{pipelineName}/run").To(n.runPipeline). Doc("run pipeline"). Reads(apis.RunPipelineRequest{}). - Returns(200, "OK", apis.PipelineRunMeta{}). + Returns(200, "OK", apis.PipelineRun{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline", "run")). Writes(apis.PipelineRunMeta{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.GET("/{pipelineName}/runs").To(p.listPipelineRuns). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs").To(n.listPipelineRuns). Doc("list pipeline runs"). Param(ws.QueryParameter("status", "query identifier of the status").DataType("string")). Returns(200, "OK", apis.ListPipelineRunResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "list")). Writes(apis.ListPipelineRunResponse{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.POST("/{pipelineName}/runs/{runName}/stop").To(p.stopPipeline). + ws.Route(ws.POST("/{projectName}/pipelines/{pipelineName}/runs/{runName}/stop").To(n.stopPipeline). Doc("stop pipeline run"). Returns(200, "OK", apis.PipelineRunMeta{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "stop")). Writes(apis.PipelineRunMeta{}).Do(meta, projParam, pipelineParam, runParam)) - ws.Route(ws.GET("/{pipelineName}/runs/{runName}").To(p.getPipelineRun). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs/{runName}").To(n.getPipelineRun). Doc("get pipeline run"). Returns(200, "OK", apis.PipelineRunBase{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "get")). Writes(apis.PipelineRunBase{}).Do(meta, projParam, pipelineParam, runParam)) - ws.Route(ws.DELETE("/{pipelineName}/runs/{runName}").To(p.deletePipelineRun). + ws.Route(ws.DELETE("/{projectName}/pipelines/{pipelineName}/runs/{runName}").To(n.deletePipelineRun). Doc("delete pipeline run"). Returns(200, "OK", apis.PipelineRunMeta{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "delete")). Writes(apis.PipelineRunMeta{}).Do(meta, projParam, pipelineParam, runParam)) // get pipeline run status - ws.Route(ws.GET("/{pipelineName}/runs/{runName}/status").To(p.getPipelineRunStatus). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs/{runName}/status").To(n.getPipelineRunStatus). Doc("get pipeline run status"). - Returns(200, "OK", workflowv1alpha1.WorkflowRunStatus{}). + Returns(200, "OK", v1alpha1.WorkflowRunStatus{}). Returns(400, "Bad Request", bcode.Bcode{}). - Writes(workflowv1alpha1.WorkflowRunStatus{}).Do(meta, projParam, pipelineParam, runParam)) + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "detail")). + Writes(v1alpha1.WorkflowRunStatus{}).Do(meta, projParam, pipelineParam, runParam)) // get pipeline run log - ws.Route(ws.GET("/{pipelineName}/runs/{runName}/log").To(p.getPipelineRunLog). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs/{runName}/log").To(n.getPipelineRunLog). Doc("get pipeline run log"). - Param(ws.QueryParameter("step", "query by specific id").DataType("string")). + Param(ws.QueryParameter("step", "query by specific step name").DataType("string")). Returns(200, "OK", apis.GetPipelineRunLogResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "detail")). Writes(apis.GetPipelineRunLogResponse{}).Do(meta, projParam, pipelineParam, runParam)) // get pipeline run output - ws.Route(ws.GET("/{pipelineName}/runs/{runName}/output").To(p.getPipelineRunOutput). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs/{runName}/output").To(n.getPipelineRunOutput). Doc("get pipeline run output"). - Param(ws.QueryParameter("step", "query by specific id").DataType("string")). + Param(ws.QueryParameter("step", "query by specific step name").DataType("string").Required(true)). Returns(200, "OK", apis.GetPipelineRunOutputResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "detail")). Writes(apis.GetPipelineRunOutputResponse{}).Do(meta, projParam, pipelineParam, runParam)) + // get pipeline run input + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs/{runName}/input").To(n.getPipelineRunInput). + Doc("get pipeline run input"). + Param(ws.QueryParameter("step", "query by specific step name").DataType("string").Required(true)). + Returns(200, "OK", apis.GetPipelineRunInputResponse{}). + Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "detail")). + Writes(apis.GetPipelineRunInputResponse{}).Do(meta, projParam, pipelineParam, runParam)) + + ws.Filter(authCheckFilter) +} + +// GetWebServiceRoute is the implementation of pipeline Interface +func (n *pipelineAPIInterface) GetWebServiceRoute() *restful.WebService { + tags := []string{"pipeline"} + meta := func(builder *restful.RouteBuilder) { + builder.Metadata(restfulspec.KeyOpenAPITags, tags) + } + + ws := new(restful.WebService) + + ws.Path(versionPrefix+"/pipelines"). + Consumes(restful.MIME_XML, restful.MIME_JSON). + Produces(restful.MIME_JSON, restful.MIME_XML). + Doc("api for project manage") + + ws.Route(ws.GET("").To(n.listPipelines). + Doc("list pipelines"). + Param(ws.QueryParameter("query", "Fuzzy search based on name or description").DataType("string")). + Param(ws.QueryParameter("projectName", "query pipelines within a project").DataType("string")). + Param(ws.QueryParameter("detailed", "query pipelines with detail").DataType("bool").DefaultValue("true")). + Returns(200, "OK", apis.ListPipelineResponse{}). + Returns(400, "Bad Request", bcode.Bcode{}). + Writes(apis.ListPipelineResponse{}).Do(meta)) + ws.Filter(authCheckFilter) return ws } +type pipelineAPIInterface struct { + PipelineService service.PipelineService `inject:""` +} + // NewPipelineAPIInterface new pipeline manage APIInterface func NewPipelineAPIInterface() Interface { return &pipelineAPIInterface{} } -func (p *pipelineAPIInterface) listPipelines(req *restful.Request, res *restful.Response) { - var projetNames []string - if req.QueryParameter("project") != "" { - projetNames = append(projetNames, req.QueryParameter("project")) +func (n *pipelineAPIInterface) listPipelines(req *restful.Request, res *restful.Response) { + var projectNames []string + if req.QueryParameter(Project) != "" { + projectNames = append(projectNames, req.QueryParameter(Project)) } - pipelines, err := p.PipelineService.ListPipelines(req.Request.Context(), apis.ListPipelineRequest{ - Projects: projetNames, + _detailed := req.QueryParameter("detailed") + if _detailed == "" { + _detailed = "true" + } + detailed, err := strconv.ParseBool(_detailed) + if err != nil { + bcode.ReturnError(req, res, errors.Wrap(err, "invalid detailed param")) + } + pipelines, err := n.PipelineService.ListPipelines(req.Request.Context(), apis.ListPipelineRequest{ + Projects: projectNames, Query: req.QueryParameter("query"), + Detailed: detailed, }) if err != nil { log.Logger.Errorf("list pipeline failure %s", err.Error()) @@ -223,15 +266,18 @@ func (p *pipelineAPIInterface) listPipelines(req *restful.Request, res *restful. } } -func (p *pipelineAPIInterface) getPipeline(req *restful.Request, res *restful.Response) { - pipeline := req.Request.Context().Value(apis.CtxKeyPipeline).(apis.PipelineBase) +func (n *projectAPIInterface) getPipeline(req *restful.Request, res *restful.Response) { + pipeline, err := n.PipelineService.GetPipeline(req.Request.Context(), req.PathParameter(Pipeline), true) + if err != nil { + return + } if err := res.WriteEntity(pipeline); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) createPipeline(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) createPipeline(req *restful.Request, res *restful.Response) { var createReq apis.CreatePipelineRequest if err := req.ReadEntity(&createReq); err != nil { bcode.ReturnError(req, res, err) @@ -241,19 +287,25 @@ func (p *pipelineAPIInterface) createPipeline(req *restful.Request, res *restful bcode.ReturnError(req, res, err) return } - pipelineBase, err := p.PipelineService.CreatePipeline(req.Request.Context(), createReq) + pipelineBase, err := n.PipelineService.CreatePipeline(req.Request.Context(), createReq) if err != nil { log.Logger.Errorf("create pipeline failure %s", err.Error()) bcode.ReturnError(req, res, err) return } + _, err = n.ContextService.InitContext(req.Request.Context(), pipelineBase.Project.Name, pipelineBase.Name) + if err != nil { + log.Logger.Errorf("init pipeline context failure: %s", err.Error()) + bcode.ReturnError(req, res, err) + return + } if err := res.WriteEntity(pipelineBase); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) updatePipeline(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) updatePipeline(req *restful.Request, res *restful.Response) { var updateReq apis.UpdatePipelineRequest if err := req.ReadEntity(&updateReq); err != nil { bcode.ReturnError(req, res, err) @@ -263,8 +315,8 @@ func (p *pipelineAPIInterface) updatePipeline(req *restful.Request, res *restful bcode.ReturnError(req, res, err) return } - base := req.Request.Context().Value(apis.CtxKeyPipeline).(apis.PipelineBase) - pipelineBase, err := p.PipelineService.UpdatePipeline(req.Request.Context(), base.Name, base.Project, updateReq) + pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) + pipelineBase, err := n.PipelineService.UpdatePipeline(req.Request.Context(), pipeline.Name, updateReq) if err != nil { log.Logger.Errorf("update pipeline failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -276,43 +328,43 @@ func (p *pipelineAPIInterface) updatePipeline(req *restful.Request, res *restful } } -func (p *pipelineAPIInterface) deletePipeline(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) deletePipeline(req *restful.Request, res *restful.Response) { pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) - err := p.PipelineService.DeletePipeline(req.Request.Context(), pipeline) + err := n.PipelineService.DeletePipeline(req.Request.Context(), pipeline) if err != nil { log.Logger.Errorf("delete pipeline failure %s", err.Error()) bcode.ReturnError(req, res, err) return } - if err := res.WriteEntity(pipeline.PipelineMeta); err != nil { + if err := res.WriteEntity(apis.EmptyResponse{}); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) runPipeline(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) runPipeline(req *restful.Request, res *restful.Response) { var runReq apis.RunPipelineRequest pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) - if err := req.ReadEntity(runReq); err != nil { + if err := req.ReadEntity(&runReq); err != nil { bcode.ReturnError(req, res, err) return } - err := p.PipelineService.RunPipeline(req.Request.Context(), pipeline, runReq) + run, err := n.PipelineService.RunPipeline(req.Request.Context(), pipeline, runReq) if err != nil { log.Logger.Errorf("run pipeline failure %s", err.Error()) bcode.ReturnError(req, res, err) return } - if err := res.WriteEntity(pipeline.PipelineMeta); err != nil { + if err := res.WriteEntity(run); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) stopPipeline(req *restful.Request, res *restful.Response) { - pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(apis.PipelineRun) - err := p.PipelineRunService.StopPipelineRun(req.Request.Context(), pipelineRun.PipelineRunBase) +func (n *projectAPIInterface) stopPipeline(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) + err := n.PipelineRunService.StopPipelineRun(req.Request.Context(), pipelineRun.PipelineRunBase) if err != nil { log.Logger.Errorf("stop pipeline failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -324,9 +376,9 @@ func (p *pipelineAPIInterface) stopPipeline(req *restful.Request, res *restful.R } } -func (p *pipelineAPIInterface) listPipelineRuns(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) listPipelineRuns(req *restful.Request, res *restful.Response) { pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) - pipelineRuns, err := p.PipelineRunService.ListPipelineRuns(req.Request.Context(), pipeline) + pipelineRuns, err := n.PipelineRunService.ListPipelineRuns(req.Request.Context(), pipeline) if err != nil { log.Logger.Errorf("list pipeline runs failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -338,33 +390,65 @@ func (p *pipelineAPIInterface) listPipelineRuns(req *restful.Request, res *restf } } -func (p *pipelineAPIInterface) getPipelineRun(req *restful.Request, res *restful.Response) { - pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(apis.PipelineRun) +func (n *projectAPIInterface) getPipelineRun(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) if err := res.WriteEntity(pipelineRun.PipelineRunBase); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) getPipelineRunStatus(req *restful.Request, res *restful.Response) { - pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(apis.PipelineRun) +func (n *projectAPIInterface) getPipelineRunStatus(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) if err := res.WriteEntity(pipelineRun.Status); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) getPipelineRunLog(req *restful.Request, res *restful.Response) { - +func (n *projectAPIInterface) getPipelineRunLog(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) + logs, err := n.PipelineRunService.GetPipelineRunLog(req.Request.Context(), *pipelineRun, req.QueryParameter("step")) + if err != nil { + log.Logger.Errorf("get pipeline run log failure %s", err.Error()) + bcode.ReturnError(req, res, err) + return + } + if err := res.WriteEntity(logs); err != nil { + bcode.ReturnError(req, res, err) + return + } } -func (p *pipelineAPIInterface) getPipelineRunOutput(req *restful.Request, res *restful.Response) { - +func (n *projectAPIInterface) getPipelineRunOutput(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) + output, err := n.PipelineRunService.GetPipelineRunOutput(req.Request.Context(), *pipelineRun, req.QueryParameter("step")) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + if err := res.WriteEntity(output); err != nil { + bcode.ReturnError(req, res, err) + return + } } -func (p *pipelineAPIInterface) deletePipelineRun(req *restful.Request, res *restful.Response) { - pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(apis.PipelineRun) - err := p.PipelineRunService.DeletePipelineRun(req.Request.Context(), pipelineRun.PipelineRunMeta) +func (n *projectAPIInterface) getPipelineRunInput(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) + input, err := n.PipelineRunService.GetPipelineRunInput(req.Request.Context(), *pipelineRun, req.QueryParameter("step")) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + if err := res.WriteEntity(input); err != nil { + bcode.ReturnError(req, res, err) + return + } +} + +func (n *projectAPIInterface) deletePipelineRun(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) + err := n.PipelineRunService.DeletePipelineRun(req.Request.Context(), pipelineRun.PipelineRunMeta) if err != nil { log.Logger.Errorf("delete pipeline run failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -376,11 +460,11 @@ func (p *pipelineAPIInterface) deletePipelineRun(req *restful.Request, res *rest } } -func (p *pipelineAPIInterface) listContextValues(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) listContextValues(req *restful.Request, res *restful.Response) { pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) - contextValues, err := p.ContextService.ListContexts(req.Request.Context(), pipeline.Project, pipeline.Name) + contextValues, err := n.ContextService.ListContexts(req.Request.Context(), pipeline.Project.Name, pipeline.Name) if err != nil { - log.Logger.Errorf("list context values failure %s", err.Error()) + log.Logger.Errorf("list context values failure: %s", err.Error()) bcode.ReturnError(req, res, err) return } @@ -390,7 +474,7 @@ func (p *pipelineAPIInterface) listContextValues(req *restful.Request, res *rest } } -func (p *pipelineAPIInterface) createContextValue(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) createContextValue(req *restful.Request, res *restful.Response) { pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) var createReq apis.CreateContextValuesRequest if err := req.ReadEntity(&createReq); err != nil { @@ -403,7 +487,7 @@ func (p *pipelineAPIInterface) createContextValue(req *restful.Request, res *res } pipelineCtx := apis.Context(createReq) - _, err := p.ContextService.CreateContext(req.Request.Context(), pipeline.Project, pipeline.Name, pipelineCtx) + _, err := n.ContextService.CreateContext(req.Request.Context(), pipeline.Project.Name, pipeline.Name, pipelineCtx) if err != nil { log.Logger.Errorf("create context failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -415,8 +499,8 @@ func (p *pipelineAPIInterface) createContextValue(req *restful.Request, res *res } } -func (p *pipelineAPIInterface) updateContextValue(req *restful.Request, res *restful.Response) { - plCtx := req.Request.Context().Value(&apis.CtxKeyPipelineContex).(apis.Context) +func (n *projectAPIInterface) updateContextValue(req *restful.Request, res *restful.Response) { + plCtx := req.Request.Context().Value(&apis.CtxKeyPipelineContext).(apis.Context) pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) var updateReq apis.UpdateContextValuesRequest if err := req.ReadEntity(&updateReq); err != nil { @@ -428,7 +512,7 @@ func (p *pipelineAPIInterface) updateContextValue(req *restful.Request, res *res return } pipelineCtx := apis.Context{Name: plCtx.Name, Values: updateReq.Values} - _, err := p.ContextService.UpdateContext(req.Request.Context(), pipeline.Project, pipeline.Name, pipelineCtx) + _, err := n.ContextService.UpdateContext(req.Request.Context(), pipeline.Project.Name, pipeline.Name, pipelineCtx) if err != nil { log.Logger.Errorf("update context failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -440,10 +524,10 @@ func (p *pipelineAPIInterface) updateContextValue(req *restful.Request, res *res } } -func (p *pipelineAPIInterface) deleteContextValue(req *restful.Request, res *restful.Response) { - plCtx := req.Request.Context().Value(&apis.CtxKeyPipelineContex).(apis.Context) +func (n *projectAPIInterface) deleteContextValue(req *restful.Request, res *restful.Response) { + plCtx := req.Request.Context().Value(&apis.CtxKeyPipelineContext).(apis.Context) pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) - err := p.ContextService.DeleteContext(req.Request.Context(), pipeline.Project, pipeline.Name, plCtx.Name) + err := n.ContextService.DeleteContext(req.Request.Context(), pipeline.Project.Name, pipeline.Name, plCtx.Name) if err != nil { log.Logger.Errorf("delete context failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -455,8 +539,18 @@ func (p *pipelineAPIInterface) deleteContextValue(req *restful.Request, res *res } } -func (p *pipelineAPIInterface) pipelineCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { - pipeline, err := p.PipelineService.GetPipeline(req.Request.Context(), req.PathParameter("pipelineName"), req.QueryParameter("projectName")) +func (n *projectAPIInterface) projectCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { + project, err := n.ProjectService.GetProject(req.Request.Context(), req.PathParameter(Project)) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + req.Request = req.Request.WithContext(context.WithValue(req.Request.Context(), &apis.CtxKeyProject, project)) + chain.ProcessFilter(req, res) +} + +func (n *projectAPIInterface) pipelineCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { + pipeline, err := n.PipelineService.GetPipeline(req.Request.Context(), req.PathParameter(Pipeline), false) if err != nil { bcode.ReturnError(req, res, err) return @@ -465,8 +559,8 @@ func (p *pipelineAPIInterface) pipelineCheckFilter(req *restful.Request, res *re chain.ProcessFilter(req, res) } -func (p *pipelineAPIInterface) pipelineContextCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { - contexts, err := p.ContextService.ListContexts(req.Request.Context(), req.PathParameter("pipelineName"), req.QueryParameter("projectName")) +func (n *projectAPIInterface) pipelineContextCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { + contexts, err := n.ContextService.ListContexts(req.Request.Context(), req.PathParameter(Project), req.PathParameter(Pipeline)) if err != nil { bcode.ReturnError(req, res, err) return @@ -477,20 +571,22 @@ func (p *pipelineAPIInterface) pipelineContextCheckFilter(req *restful.Request, bcode.ReturnError(req, res, bcode.ErrContextNotFound) return } - req.Request = req.Request.WithContext(context.WithValue(req.Request.Context(), &apis.CtxKeyPipelineContex, apis.Context{ + req.Request = req.Request.WithContext(context.WithValue(req.Request.Context(), &apis.CtxKeyPipelineContext, apis.Context{ Name: contextName, Values: contextValue, })) chain.ProcessFilter(req, res) } -func (p *pipelineAPIInterface) pipelineRunCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { +func (n *projectAPIInterface) pipelineRunCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { meta := apis.PipelineRunMeta{ - PipelineName: req.PathParameter(string(Pipeline)), - Project: req.QueryParameter(string(Project)), - PipelineRunName: req.PathParameter(string(PipelineRun)), + PipelineName: req.PathParameter(Pipeline), + Project: apis.NameAlias{ + Name: req.PathParameter(Project), + }, + PipelineRunName: req.PathParameter(PipelineRun), } - run, err := p.PipelineRunService.GetPipelineRun(req.Request.Context(), meta) + run, err := n.PipelineRunService.GetPipelineRun(req.Request.Context(), meta) if err != nil { bcode.ReturnError(req, res, err) return diff --git a/pkg/apiserver/interfaces/api/project.go b/pkg/apiserver/interfaces/api/project.go index 8a7c548de..032d5bda4 100644 --- a/pkg/apiserver/interfaces/api/project.go +++ b/pkg/apiserver/interfaces/api/project.go @@ -29,10 +29,14 @@ import ( ) type projectAPIInterface struct { - RbacService service.RBACService `inject:""` - ProjectService service.ProjectService `inject:""` - TargetService service.TargetService `inject:""` - ConfigService service.ConfigService `inject:""` + RbacService service.RBACService `inject:""` + ProjectService service.ProjectService `inject:""` + TargetService service.TargetService `inject:""` + ConfigService service.ConfigService `inject:""` + PipelineService service.PipelineService `inject:""` + PipelineRunService service.PipelineRunService `inject:""` + ContextService service.ContextService `inject:""` + RBACService service.RBACService `inject:""` } // NewProjectAPIInterface new project APIInterface @@ -305,6 +309,7 @@ func (n *projectAPIInterface) GetWebServiceRoute() *restful.WebService { Returns(400, "Bad Request", bcode.Bcode{}). Writes(apis.ListTerraformProviderResponse{})) + initPipelineRoutes(ws, n) ws.Filter(authCheckFilter) return ws } diff --git a/pkg/apiserver/utils/bcode/017_pipeline.go b/pkg/apiserver/utils/bcode/017_pipeline.go index 6b31f5ad6..635b0cf95 100644 --- a/pkg/apiserver/utils/bcode/017_pipeline.go +++ b/pkg/apiserver/utils/bcode/017_pipeline.go @@ -21,4 +21,22 @@ var ( ErrContextNotFound = NewBcode(400, 17001, "pipeline context is not found") // ErrContextAlreadyExist means the certain context already exists ErrContextAlreadyExist = NewBcode(400, 17002, "pipeline context of pipeline already exist") + // ErrGetPipelineInfo means failed to get pipeline info + ErrGetPipelineInfo = NewBcode(400, 17003, "get pipeline info failed") + // ErrPipelineNotExist means specific pipeline not found + ErrPipelineNotExist = NewBcode(404, 17004, "failed to find log pods") + // ErrGetPodsLogs means failed to get pods logs + ErrGetPodsLogs = NewBcode(500, 17006, "failed to get pods logs") + // ErrReadSourceLog means failed to read source log + ErrReadSourceLog = NewBcode(500, 17007, "failed to read log from URL source") + // ErrGetContextBackendData means failed to get context backend data + ErrGetContextBackendData = NewBcode(500, 17008, "failed to get context backend data") + // ErrNoSteps means pipeline doesn't have a step + ErrNoSteps = NewBcode(400, 17009, "pipeline step number is zero") + // ErrPipelineExist means the pipeline is exist + ErrPipelineExist = NewBcode(400, 17010, "the pipeline is exist") + // ErrPipelineRunFinished means pipeline run is finished + ErrPipelineRunFinished = NewBcode(400, 17011, "pipeline run is finished") + // ErrWrongMode means the pipeline run mode is wrong + ErrWrongMode = NewBcode(400, 17012, "wrong pipeline run mode, only \"DAG\" and \"StepByStep\" are supported") ) diff --git a/pkg/utils/k8s.go b/pkg/utils/k8s.go index 4eba13c82..948ffab68 100644 --- a/pkg/utils/k8s.go +++ b/pkg/utils/k8s.go @@ -21,7 +21,19 @@ import ( "encoding/json" "fmt" "os" + "regexp" "strings" + "text/template" + "time" + + "github.com/fatih/color" + "github.com/pkg/errors" + "github.com/wercker/stern/stern" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/client-go/kubernetes" + + querytypes "github.com/oam-dev/kubevela/pkg/velaql/providers/query/types" authv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" @@ -199,3 +211,125 @@ func IsClusterScope(gvk schema.GroupVersionKind, mapper meta.RESTMapper) (bool, isClusterScope := len(mappings) > 0 && mappings[0].Scope.Name() == meta.RESTScopeNameRoot return isClusterScope, err } + +// GetPodsLogs get logs from pods +func GetPodsLogs(ctx context.Context, config *rest.Config, containerName string, selectPods []*querytypes.PodBase, tmpl string, logC chan<- string, tailLines *int64) error { + if err := verifyPods(selectPods); err != nil { + return err + } + podRegex := getPodRegex(selectPods) + pods, err := regexp.Compile(podRegex) + if err != nil { + return fmt.Errorf("fail to compile '%s' for logs query", podRegex) + } + container := regexp.MustCompile(".*") + if containerName != "" { + container = regexp.MustCompile(containerName + ".*") + } + // These pods are from the same namespace, so we can use the first one to get the namespace + namespace := selectPods[0].Metadata.Namespace + selector := labels.NewSelector() + // Only use the labels to select pod if query one pod's log. It is only used when query vela-core log + if len(selectPods) == 1 { + for k, v := range selectPods[0].Metadata.Labels { + req, _ := labels.NewRequirement(k, selection.Equals, []string{v}) + if req != nil { + selector = selector.Add(*req) + } + } + } + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + return err + } + added, removed, err := stern.Watch(ctx, + clientSet.CoreV1().Pods(namespace), + pods, + container, + nil, + []stern.ContainerState{stern.RUNNING, stern.TERMINATED}, + selector, + ) + if err != nil { + return err + } + tails := make(map[string]*stern.Tail) + + funs := map[string]interface{}{ + "json": func(in interface{}) (string, error) { + b, err := json.Marshal(in) + if err != nil { + return "", err + } + return string(b), nil + }, + "color": func(color color.Color, text string) string { + return color.SprintFunc()(text) + }, + } + template, err := template.New("log").Funcs(funs).Parse(tmpl) + if err != nil { + return errors.Wrap(err, "unable to parse template") + } + + go func() { + for p := range added { + id := p.GetID() + if tails[id] != nil { + continue + } + // 48h + dur, _ := time.ParseDuration("48h") + tail := stern.NewTail(p.Namespace, p.Pod, p.Container, template, &stern.TailOptions{ + Timestamps: true, + SinceSeconds: int64(dur.Seconds()), + Exclude: nil, + Include: nil, + Namespace: false, + TailLines: tailLines, // default for all logs + }) + tails[id] = tail + + tail.Start(ctx, clientSet.CoreV1().Pods(p.Namespace), logC) + } + }() + + go func() { + for p := range removed { + id := p.GetID() + if tails[id] == nil { + continue + } + tails[id].Close() + delete(tails, id) + } + }() + + <-ctx.Done() + close(logC) + return nil +} + +func getPodRegex(pods []*querytypes.PodBase) string { + var podNames []string + for _, pod := range pods { + podNames = append(podNames, fmt.Sprintf("(%s.*)", pod.Metadata.Name)) + } + return strings.Join(podNames, "|") +} + +func verifyPods(pods []*querytypes.PodBase) error { + if len(pods) == 0 { + return errors.New("no pods selected") + } + if len(pods) == 1 { + return nil + } + namespace := pods[0].Metadata.Namespace + for _, pod := range pods { + if pod.Metadata.Namespace != namespace { + return errors.New("cannot select pods from different namespaces") + } + } + return nil +} diff --git a/references/cli/logs.go b/references/cli/logs.go index d1b012eea..1fd5d23ba 100644 --- a/references/cli/logs.go +++ b/references/cli/logs.go @@ -18,24 +18,16 @@ package cli import ( "context" - "encoding/json" "fmt" - "regexp" "strings" - "text/template" - "time" "github.com/fatih/color" - "github.com/pkg/errors" "github.com/spf13/cobra" - "github.com/wercker/stern/stern" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/selection" - "k8s.io/client-go/kubernetes" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/multicluster" + "github.com/oam-dev/kubevela/pkg/utils" "github.com/oam-dev/kubevela/pkg/utils/common" "github.com/oam-dev/kubevela/pkg/utils/util" querytypes "github.com/oam-dev/kubevela/pkg/velaql/providers/query/types" @@ -99,45 +91,25 @@ type Args struct { } func (l *Args) printPodLogs(ctx context.Context, ioStreams util.IOStreams, selectPod *querytypes.PodBase, filters []string) error { - pod, err := regexp.Compile(selectPod.Metadata.Name + ".*") - if err != nil { - return fmt.Errorf("fail to compile '%s' for logs query", selectPod.Metadata.Name+".*") - } - container := regexp.MustCompile(".*") - if l.ContainerName != "" { - container = regexp.MustCompile(l.ContainerName + ".*") - } - namespace := selectPod.Metadata.Namespace - selector := labels.NewSelector() - for k, v := range selectPod.Metadata.Labels { - req, _ := labels.NewRequirement(k, selection.Equals, []string{v}) - if req != nil { - selector = selector.Add(*req) - } - } - config, err := l.Args.GetConfig() if err != nil { return err } - clientSet, err := kubernetes.NewForConfig(config) - if err != nil { - return err - } - added, removed, err := stern.Watch(ctx, - clientSet.CoreV1().Pods(namespace), - pod, - container, - nil, - []stern.ContainerState{stern.RUNNING, stern.TERMINATED}, - selector, - ) - if err != nil { - return err - } - tails := make(map[string]*stern.Tail) logC := make(chan string, 1024) + var t string + switch l.Output { + case "default": + if color.NoColor { + t = "{{.ContainerName}} {{.Message}}" + } else { + t = "{{color .ContainerColor .ContainerName}} {{.Message}}" + } + case "raw": + t = "{{.Message}}" + case "json": + t = "{{json .}}\n" + } go func() { for { select { @@ -158,71 +130,11 @@ func (l *Args) printPodLogs(ctx context.Context, ioStreams util.IOStreams, selec } }() - var t string - switch l.Output { - case "default": - if color.NoColor { - t = "{{.ContainerName}} {{.Message}}" - } else { - t = "{{color .ContainerColor .ContainerName}} {{.Message}}" - } - case "raw": - t = "{{.Message}}" - case "json": - t = "{{json .}}\n" - } - funs := map[string]interface{}{ - "json": func(in interface{}) (string, error) { - b, err := json.Marshal(in) - if err != nil { - return "", err - } - return string(b), nil - }, - "color": func(color color.Color, text string) string { - return color.SprintFunc()(text) - }, - } - template, err := template.New("log").Funcs(funs).Parse(t) + err = utils.GetPodsLogs(ctx, config, l.ContainerName, []*querytypes.PodBase{selectPod}, t, logC, nil) if err != nil { - return errors.Wrap(err, "unable to parse template") + return err } - go func() { - for p := range added { - id := p.GetID() - if tails[id] != nil { - continue - } - // 48h - dur, _ := time.ParseDuration("48h") - tail := stern.NewTail(p.Namespace, p.Pod, p.Container, template, &stern.TailOptions{ - Timestamps: true, - SinceSeconds: int64(dur.Seconds()), - Exclude: nil, - Include: nil, - Namespace: false, - TailLines: nil, // default for all logs - }) - tails[id] = tail - - tail.Start(ctx, clientSet.CoreV1().Pods(p.Namespace), logC) - } - }() - - go func() { - for p := range removed { - id := p.GetID() - if tails[id] == nil { - continue - } - tails[id].Close() - delete(tails, id) - } - }() - - <-ctx.Done() - return nil } diff --git a/test/e2e-apiserver-test/pipeline_test.go b/test/e2e-apiserver-test/pipeline_test.go new file mode 100644 index 000000000..de5b68bc7 --- /dev/null +++ b/test/e2e-apiserver-test/pipeline_test.go @@ -0,0 +1,363 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_apiserver_test + +import ( + "context" + "net/http" + "strconv" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/kubevela/workflow/api/v1alpha1" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + "github.com/oam-dev/kubevela/pkg/apiserver/domain/model" + apisv1 "github.com/oam-dev/kubevela/pkg/apiserver/interfaces/api/dto/v1" + "github.com/oam-dev/kubevela/pkg/oam/util" + "github.com/oam-dev/kubevela/pkg/utils/common" +) + +var testPipelineSteps []v1alpha1.WorkflowStep + +func init() { + rawProps := []byte(`{"url":"https://api.github.com/repos/kubevela/kubevela"}`) + testPipelineSteps = []v1alpha1.WorkflowStep{ + { + WorkflowStepBase: v1alpha1.WorkflowStepBase{ + Name: "request", + Type: "request", + Outputs: v1alpha1.StepOutputs{ + { + ValueFrom: "import \"strconv\"\n\"Current star count: \" + strconv.FormatInt(response[\"stargazers_count\"], 10)\n", + Name: "stars", + }, + }, + Properties: &runtime.RawExtension{ + Raw: rawProps, + }, + }, + }, + } +} + +var _ = Describe("Test the rest api about the pipeline", func() { + var ( + projectName1 = testNSprefix + strconv.FormatInt(time.Now().UnixNano(), 10) + pipelineName = "test-pipeline" + description = "amazing pipeline" + contextName = "test-context" + contextKey = "test-key" + contextVal = "test-val" + pipelineRunName string + ) + defer GinkgoRecover() + It("create project and apply definitions", func() { + defer GinkgoRecover() + var req = apisv1.CreateProjectRequest{ + Name: projectName1, + Description: "KubeVela Project", + } + res := post("/projects", req) + var projectBase apisv1.ProjectBase + Expect(decodeResponseBody(res, &projectBase)).Should(Succeed()) + Expect(cmp.Diff(projectBase.Name, req.Name)).Should(BeEmpty()) + Expect(cmp.Diff(projectBase.Description, req.Description)).Should(BeEmpty()) + + def1 := new(v1beta1.WorkflowStepDefinition) + def2 := new(v1beta1.WorkflowStepDefinition) + Expect(common.ReadYamlToObject("./testdata/request.yaml", def1)).Should(BeNil()) + Expect(k8sClient.Create(context.Background(), def1)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + Expect(common.ReadYamlToObject("./testdata/log.yaml", def2)).Should(BeNil()) + Expect(k8sClient.Create(context.Background(), def2)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + }) + + It("create pipeline", func() { + var req = apisv1.CreatePipelineRequest{ + Name: pipelineName, + Description: description, + Spec: v1alpha1.WorkflowSpec{ + Steps: testPipelineSteps, + }, + } + res := post("/projects/"+projectName1+"/pipelines", req) + var pipeline apisv1.PipelineBase + Expect(decodeResponseBody(res, &pipeline)).Should(Succeed()) + Expect(cmp.Diff(pipeline.Name, req.Name)).Should(BeEmpty()) + Expect(len(pipeline.Spec.Steps)).Should(Equal(len(req.Spec.Steps))) + }) + + It("create context", func() { + var req = apisv1.CreateContextValuesRequest{ + Name: contextName, + Values: []model.Value{ + { + Key: contextKey, + Value: contextVal, + }, + }, + } + res := post("/projects/"+projectName1+"/pipelines/"+pipelineName+"/contexts", req) + var context apisv1.Context + Expect(decodeResponseBody(res, &context)).Should(Succeed()) + Expect(cmp.Diff(context.Name, req.Name)).Should(BeEmpty()) + Expect(cmp.Diff(context.Values, req.Values)).Should(BeEmpty()) + }) + + It("get contexts", func() { + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/contexts") + var contexs apisv1.ListContextValueResponse + Expect(decodeResponseBody(res, &contexs)).Should(Succeed()) + Expect(len(contexs.Contexts)).Should(Equal(1)) + ctx, ok := contexs.Contexts[contextName] + Expect(ok).Should(BeTrue()) + Expect(len(ctx)).Should(Equal(1)) + }) + + It("update context", func() { + var req = apisv1.UpdateContextValuesRequest{ + Values: []model.Value{ + { + Key: contextKey, + Value: "new-val", + }, + }, + } + res := put("/projects/"+projectName1+"/pipelines/"+pipelineName+"/contexts/"+contextName, req) + var context apisv1.Context + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + Expect(decodeResponseBody(res, &context)).Should(Succeed()) + + By("check the context value") + Expect(cmp.Diff(context.Values[0].Value, "new-val")).Should(BeEmpty()) + }) + + It("update pipeline", func() { + rawProps := []byte(`{"url":"https://api.github.com/repos/kubevela/kubevela"}`) + newSteps := make([]v1alpha1.WorkflowStep, 0) + newSteps = append(newSteps, v1alpha1.WorkflowStep{ + SubSteps: []v1alpha1.WorkflowStepBase{ + { + Name: "request1", + Type: "request", + Outputs: v1alpha1.StepOutputs{ + { + ValueFrom: "import \"strconv\"\n\"Current star count: \" + strconv.FormatInt(response[\"stargazers_count\"], 10)\n", + Name: "stars", + }, + }, + Properties: &runtime.RawExtension{ + Raw: rawProps, + }, + }, + { + Name: "request2", + Type: "request", + Outputs: v1alpha1.StepOutputs{ + { + ValueFrom: "import \"strconv\"\n\"Current star count: \" + strconv.FormatInt(response[\"stargazers_count\"], 10)\n", + Name: "stars-copy", + }, + }, + Properties: &runtime.RawExtension{ + Raw: rawProps, + }, + }, + }, + WorkflowStepBase: v1alpha1.WorkflowStepBase{ + Name: "request-group", + Type: "step-group", + }, + }) + newSteps = append(newSteps, v1alpha1.WorkflowStep{ + WorkflowStepBase: v1alpha1.WorkflowStepBase{ + Name: "log", + Type: "log", + Inputs: v1alpha1.StepInputs{ + { + ParameterKey: "data", + From: "stars", + }, + }, + }, + }) + var req = apisv1.UpdatePipelineRequest{ + Description: description, + Spec: v1alpha1.WorkflowSpec{ + Steps: newSteps, + }, + } + res := put("/projects/"+projectName1+"/pipelines/"+pipelineName, req) + var pipeline apisv1.PipelineBase + Expect(decodeResponseBody(res, &pipeline)).Should(Succeed()) + Expect(len(pipeline.Spec.Steps)).Should(Equal(len(req.Spec.Steps))) + }) + + It("run pipeline", func() { + var req = apisv1.RunPipelineRequest{ + Mode: v1alpha1.WorkflowExecuteMode{ + Steps: "StepByStep", + SubSteps: "DAG", + }, + ContextName: contextName, + } + res := post("/projects/"+projectName1+"/pipelines/"+pipelineName+"/run", req) + var run apisv1.PipelineRun + Expect(decodeResponseBody(res, &run)).Should(Succeed()) + Expect(run.PipelineRunName).ShouldNot(BeEmpty()) + pipelineRunName = run.PipelineRunName + }) + + It("list pipeline", func() { + res := get("/pipelines?query=amazing") + var pipelines apisv1.ListPipelineResponse + Expect(decodeResponseBody(res, &pipelines)).Should(Succeed()) + Expect(pipelines.Total).Should(BeNumerically("==", 1)) + Expect(pipelines.Pipelines[0].Name).Should(Equal(pipelineName)) + }) + + It("get pipeline", func() { + Eventually(func(g Gomega) { + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName) + var pipeline apisv1.GetPipelineResponse + g.Expect(decodeResponseBody(res, &pipeline)).Should(Succeed()) + g.Expect(pipeline.Name).Should(Equal(pipelineName)) + g.Expect(pipeline.Description).Should(Equal(description)) + g.Expect(pipeline.PipelineInfo.LastRun).ShouldNot(BeNil()) + g.Expect(pipeline.PipelineInfo.RunStat.Total).Should(Equal(apisv1.RunStatInfo{Total: 1, Success: 1})) + g.Expect(len(pipeline.PipelineInfo.RunStat.Week)).Should(Equal(7)) + }, 10*time.Second, 1*time.Second).Should(Succeed()) + }) + + It("list pipeline runs", func() { + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs") + var runs apisv1.ListPipelineRunResponse + Expect(decodeResponseBody(res, &runs)).Should(Succeed()) + Expect(runs.Total).Should(BeNumerically("==", 1)) + }) + + It("get pipeline run", func() { + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName) + var run apisv1.PipelineRunBase + Expect(decodeResponseBody(res, &run)).Should(Succeed()) + Expect(run.PipelineRunName).Should(Equal(pipelineRunName)) + }) + + It("get pipeline run status", func() { + Eventually(func(g Gomega) { + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName + "/status") + var status v1alpha1.WorkflowRunStatus + g.Expect(decodeResponseBody(res, &status)).Should(Succeed()) + g.Expect(status.Finished).Should(Equal(true)) + g.Expect(status.Phase).Should(Equal(v1alpha1.WorkflowStateSucceeded)) + g.Expect(status.Message).Should(BeEmpty()) + }, 100*time.Second, 1*time.Second).Should(Succeed()) + }) + + It("get pipeline run output", func() { + outputStep := "request1" + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName + "/output?step=" + outputStep) + var output apisv1.GetPipelineRunOutputResponse + Expect(decodeResponseBody(res, &output)).Should(Succeed()) + Expect(output.StepOutputs).Should(HaveLen(1)) + Expect(output.StepOutputs[0].Name).Should(Equal(outputStep)) + Expect(output.StepOutputs[0].Values).Should(HaveLen(1)) + Expect(output.StepOutputs[0].Values[0].Value).ShouldNot(BeEmpty()) + }) + + It("get pipeline run input", func() { + inputStep := "log" + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName + "/input?step=" + inputStep) + var input apisv1.GetPipelineRunInputResponse + Expect(decodeResponseBody(res, &input)).Should(Succeed()) + Expect(input.StepInputs).Should(HaveLen(1)) + Expect(input.StepInputs[0].Name).Should(Equal(inputStep)) + Expect(input.StepInputs[0].Values).Should(HaveLen(1)) + Expect(input.StepInputs[0].Values[0].Value).ShouldNot(BeEmpty()) + }) + + It("get pipeline run logs", func() { + logStep := "log" + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName + "/log?step=" + logStep) + var logs apisv1.GetPipelineRunLogResponse + Expect(decodeResponseBody(res, &logs)).Should(Succeed()) + Expect(logs.Name).Should(Equal(logStep)) + Expect(logs.Log).ShouldNot(BeEmpty()) + }) + + It("delete pipeline run", func() { + res := delete("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + }) + + It("stop pipeline", func() { + By("update pipeline so that it will run for a while") + var req = apisv1.UpdatePipelineRequest{ + Spec: v1alpha1.WorkflowSpec{ + Steps: []v1alpha1.WorkflowStep{ + { + WorkflowStepBase: v1alpha1.WorkflowStepBase{ + Name: "request", + Type: "request", + Timeout: "20s", + DependsOn: []string{"not-exist-step"}, + }, + }, + }, + }, + } + res := put("/projects/"+projectName1+"/pipelines/"+pipelineName, req) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + + By("run the pipeline") + var run apisv1.PipelineRun + res = post("/projects/"+projectName1+"/pipelines/"+pipelineName+"/run", apisv1.RunPipelineRequest{}) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + Expect(decodeResponseBody(res, &run)).Should(Succeed()) + pipelineRunName = run.PipelineRunName + + By("stop the pipeline") + var meta apisv1.PipelineRunMeta + res = post("/projects/"+projectName1+"/pipelines/"+pipelineName+"/runs/"+pipelineRunName+"/stop", nil) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + Expect(decodeResponseBody(res, &meta)).Should(Succeed()) + Expect(meta.PipelineRunName).Should(Equal(pipelineRunName)) + + By("delete pipeline run") + res = delete("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + }) + + It("delete context", func() { + res := delete("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/contexts/" + contextName) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + }) + + It("delete pipeline", func() { + res := delete("/projects/" + projectName1 + "/pipelines/" + pipelineName) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + }) + + It("delete project", func() { + res := delete("/projects/" + projectName1) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + }) + +}) diff --git a/test/e2e-apiserver-test/suite_test.go b/test/e2e-apiserver-test/suite_test.go index 4f1eb3392..1aff444b3 100644 --- a/test/e2e-apiserver-test/suite_test.go +++ b/test/e2e-apiserver-test/suite_test.go @@ -59,6 +59,7 @@ func TestE2eApiserverTest(t *testing.T) { // Suite test in e2e-apiserver-test relies on the pre-setup kubernetes environment var _ = BeforeSuite(func() { + defer GinkgoRecover() ctx := context.Background() @@ -115,7 +116,7 @@ var _ = BeforeSuite(func() { err = json.NewDecoder(resp.Body).Decode(code) Expect(err).Should(BeNil()) return fmt.Errorf("rest service not ready code:%d message:%s", resp.StatusCode, code.Message) - }, time.Second*10, time.Millisecond*200).Should(BeNil()) + }, time.Second*20, time.Millisecond*200).Should(BeNil()) var err error k8sClient, err = clients.GetKubeClient() Expect(err).ShouldNot(HaveOccurred()) diff --git a/test/e2e-apiserver-test/testdata/log.yaml b/test/e2e-apiserver-test/testdata/log.yaml new file mode 100644 index 000000000..b646796cb --- /dev/null +++ b/test/e2e-apiserver-test/testdata/log.yaml @@ -0,0 +1,35 @@ +apiVersion: core.oam.dev/v1beta1 +kind: WorkflowStepDefinition +metadata: + annotations: + definition.oam.dev/alias: "" + definition.oam.dev/description: Apply raw kubernetes objects for your workflow steps + labels: + custom.definition.oam.dev/ui-hidden: "true" + name: log + namespace: vela-system +spec: + schematic: + cue: + template: | + import ( + "vela/op" + ) + + apply: op.#Log & { + parameter + } + parameter: { + data?: string + level: *3 | int + source?: close({ + url: string + }) | close({ + resources?: [...{ + name?: string + cluster?: string + namespace?: string + labelSelector?: {...} + }] + }) + } diff --git a/test/e2e-apiserver-test/testdata/request.yaml b/test/e2e-apiserver-test/testdata/request.yaml new file mode 100644 index 000000000..2aa294d28 --- /dev/null +++ b/test/e2e-apiserver-test/testdata/request.yaml @@ -0,0 +1,43 @@ +apiVersion: core.oam.dev/v1beta1 +kind: WorkflowStepDefinition +metadata: + annotations: + definition.oam.dev/alias: "" + definition.oam.dev/description: Send request to the url + name: request + namespace: vela-system +spec: + schematic: + cue: + template: | + import ( + "vela/op" + "encoding/json" + ) + + http: op.http.#Do & { + method: parameter.method + url: parameter.url + request: { + if parameter.body != _|_ { + body: json.Marshal(parameter.body) + } + if parameter.header != _|_ { + header: parameter.header + } + } + } + fail: op.#Steps & { + if http.response.statusCode > 400 { + requestFail: op.#Fail & { + message: "request of \(parameter.url) is fail: \(http.response.statusCode)" + } + } + } + response: json.Unmarshal(http.response.body) + parameter: { + url: string + method: *"GET" | "POST" | "PUT" | "DELETE" + body?: {...} + header?: [string]: string + }