From 23cbc9c91fbc9d9e7c2da2f1dba0fdf2b31b2fc1 Mon Sep 17 00:00:00 2001 From: qiaozp <47812250+chivalryq@users.noreply.github.com> Date: Mon, 31 Oct 2022 23:59:04 +0800 Subject: [PATCH] Feat: implement pipeline APIs (#4908) * add context when run pipeline Signed-off-by: Qiaozp * Feat: implement pipeline API Signed-off-by: Qiaozp * Extract get log logic and implement getPipelineRunLog API Signed-off-by: Qiaozp * Init and delete pipeline contexts Signed-off-by: Qiaozp * fix panic Signed-off-by: Qiaozp * Allow not specifying context Signed-off-by: Qiaozp * change pipeline to path parameter Signed-off-by: Qiaozp * Add permission check filter Signed-off-by: Qiaozp * project -> projects in route Signed-off-by: Qiaozp * fix route conflict Signed-off-by: Qiaozp * Add project alias Signed-off-by: Qiaozp * Feat: change the list pipeline API Signed-off-by: barnettZQG * Feat: filter the project Signed-off-by: barnettZQG * Fix: the error of the run APi Signed-off-by: barnettZQG * fix log pipeline run API Signed-off-by: Qiaozp * Fix lint, fix the error of log api Signed-off-by: Qiaozp * fix error returning Signed-off-by: Qiaozp * Fix: change the lable to annotation Signed-off-by: barnettZQG * remove log config not found error Signed-off-by: Qiaozp * fix pipeline list api return no context info Signed-off-by: Qiaozp * Fix: create the namespace Signed-off-by: barnettZQG * get pipeline lastrun info Signed-off-by: Qiaozp * allow query single step output Signed-off-by: Qiaozp * organize code in api layer Signed-off-by: Qiaozp * fix project filter, add context value when get pp run, extend lastRun Signed-off-by: Qiaozp * fix get output and implement get input api Signed-off-by: Qiaozp * Fix: change the last run Signed-off-by: barnettZQG * if query sub-step outout, return it directly Signed-off-by: Qiaozp * Fix: change the run stats Signed-off-by: barnettZQG * Fix: change the output Signed-off-by: barnettZQG * flatten the input/output api Signed-off-by: Qiaozp * more info for i/o vars Signed-off-by: Qiaozp * fix nested i/o struct Signed-off-by: Qiaozp * add fromStep in input api Signed-off-by: Qiaozp * add e2e test skeleton Signed-off-by: Qiaozp * add more e2e test Signed-off-by: Qiaozp * use db to store pipeline Signed-off-by: Qiaozp * keep the last 5k lines of log Signed-off-by: Qiaozp * use stern param to keep last lines of logs Signed-off-by: Qiaozp * filter, nil labels, spec check Signed-off-by: Qiaozp * empty res, index, detail param Signed-off-by: Qiaozp * Add e2e test Signed-off-by: Qiaozp * fix e2e test and unit test Signed-off-by: Qiaozp * add context e2e test Signed-off-by: Qiaozp * goimports Signed-off-by: Qiaozp * add more test Signed-off-by: Qiaozp * review Signed-off-by: Qiaozp * remove optional tag in returned value, unify the imports name Signed-off-by: Qiaozp * fix e2e test Signed-off-by: Qiaozp * add stop test Signed-off-by: Qiaozp * more coverage Signed-off-by: Qiaozp * single case selct Signed-off-by: Qiaozp * optimize log color Signed-off-by: Qiaozp * add default permission and role Signed-off-by: Qiaozp * fix permission ut Signed-off-by: Qiaozp * change the log api implementation Signed-off-by: Qiaozp * add color, add container order Signed-off-by: Qiaozp * lint Signed-off-by: Qiaozp * fix filter nil will cut all log Signed-off-by: Qiaozp * longer timeout and lint Signed-off-by: Qiaozp Signed-off-by: Qiaozp Signed-off-by: barnettZQG Co-authored-by: barnettZQG --- .github/workflows/apiserver-test.yaml | 2 + docs/apidoc/swagger.json | 2031 +++++++++-------- e2e/addon/mock/testrepo/helm-repo/index.yaml | 11 + .../helm-repo/vela-workflow-v0.3.1.tgz | Bin 0 -> 2006 bytes e2e/addon/mock/vela_addon_mock_server.go | 15 +- go.mod | 2 +- pkg/addon/addon.go | 2 +- .../domain/model/{context.go => pipeline.go} | 44 +- pkg/apiserver/domain/model/project.go | 8 +- .../domain/service/cloudshell_test.go | 3 +- pkg/apiserver/domain/service/helm_test.go | 4 +- pkg/apiserver/domain/service/pipeline.go | 1105 +++++++-- pkg/apiserver/domain/service/pipeline_test.go | 143 ++ pkg/apiserver/domain/service/project.go | 15 + pkg/apiserver/domain/service/rbac.go | 24 +- pkg/apiserver/domain/service/rbac_test.go | 2 +- pkg/apiserver/domain/service/suite_test.go | 6 +- pkg/apiserver/domain/service/workflow_test.go | 2 +- .../infrastructure/clients/kubeclient.go | 12 +- pkg/apiserver/interfaces/api/dto/v1/types.go | 83 +- pkg/apiserver/interfaces/api/interfaces.go | 2 +- pkg/apiserver/interfaces/api/pipeline.go | 336 ++- pkg/apiserver/interfaces/api/project.go | 13 +- pkg/apiserver/utils/bcode/017_pipeline.go | 18 + pkg/utils/k8s.go | 134 ++ references/cli/logs.go | 120 +- test/e2e-apiserver-test/pipeline_test.go | 363 +++ test/e2e-apiserver-test/suite_test.go | 3 +- test/e2e-apiserver-test/testdata/log.yaml | 35 + test/e2e-apiserver-test/testdata/request.yaml | 43 + 30 files changed, 3154 insertions(+), 1427 deletions(-) create mode 100644 e2e/addon/mock/testrepo/helm-repo/vela-workflow-v0.3.1.tgz rename pkg/apiserver/domain/model/{context.go => pipeline.go} (62%) create mode 100644 pkg/apiserver/domain/service/pipeline_test.go create mode 100644 test/e2e-apiserver-test/pipeline_test.go create mode 100644 test/e2e-apiserver-test/testdata/log.yaml create mode 100644 test/e2e-apiserver-test/testdata/request.yaml diff --git a/.github/workflows/apiserver-test.yaml b/.github/workflows/apiserver-test.yaml index 8a1661cda..b44a0eb18 100644 --- a/.github/workflows/apiserver-test.yaml +++ b/.github/workflows/apiserver-test.yaml @@ -176,10 +176,12 @@ jobs: make e2e-cleanup make e2e-setup-core bin/vela addon enable fluxcd + bin/vela addon enable vela-workflow timeout 600s bash -c -- 'while true; do kubectl get ns flux-system; if [ $? -eq 0 ] ; then break; else sleep 5; fi;done' kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-core,app.kubernetes.io/instance=kubevela -n vela-system --timeout=600s kubectl wait --for=condition=Ready pod -l app=source-controller -n flux-system --timeout=600s kubectl wait --for=condition=Ready pod -l app=helm-controller -n flux-system --timeout=600s + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-workflow -n vela-system --timeout=600s - name: Run api server e2e test run: | diff --git a/docs/apidoc/swagger.json b/docs/apidoc/swagger.json index 624f824ce..9961a4213 100644 --- a/docs/apidoc/swagger.json +++ b/docs/apidoc/swagger.json @@ -4200,900 +4200,6 @@ } } }, - "/api/v1/pipelines": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "list pipelines", - "operationId": "listPipelines", - "parameters": [ - { - "type": "string", - "description": "Fuzzy search based on name or description", - "name": "query", - "in": "query" - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.ListPipelineResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "post": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "create pipeline", - "operationId": "createPipeline", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.CreatePipelineRequest" - } - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineBase" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "get pipeline", - "operationId": "getPipeline", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.GetPipelineRequest" - } - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.GetPipelineResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "put": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "update pipeline", - "operationId": "updatePipeline", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.UpdatePipelineRequest" - } - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineBase" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "delete": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "delete pipeline", - "operationId": "deletePipeline", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineMetaResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/contexts": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "list pipeline context values", - "operationId": "listContextValues", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.ListContextValueResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "post": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "create pipeline context values", - "operationId": "createContextValue", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.CreateContextValuesRequest" - } - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.Context" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/contexts/{contextName}": { - "put": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "update pipeline context value", - "operationId": "updateContextValue", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.UpdateContextValuesRequest" - } - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline context name", - "name": "contextName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.Context" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "delete": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "delete pipeline context value", - "operationId": "deleteContextValue", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline context name", - "name": "contextName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.ContextNameResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/run": { - "post": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "run pipeline", - "operationId": "runPipeline", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.RunPipelineRequest" - } - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineRunMeta" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "list pipeline runs", - "operationId": "listPipelineRuns", - "parameters": [ - { - "type": "string", - "description": "query identifier of the status", - "name": "status", - "in": "query" - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.ListPipelineRunResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs/{runName}": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "get pipeline run", - "operationId": "getPipelineRun", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineRunBase" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "delete": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "delete pipeline run", - "operationId": "deletePipelineRun", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineRunMeta" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs/{runName}/log": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "get pipeline run log", - "operationId": "getPipelineRunLog", - "parameters": [ - { - "type": "string", - "description": "query by specific id", - "name": "step", - "in": "query" - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.GetPipelineRunLogResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs/{runName}/output": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "get pipeline run output", - "operationId": "getPipelineRunOutput", - "parameters": [ - { - "type": "string", - "description": "query by specific id", - "name": "step", - "in": "query" - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.GetPipelineRunOutputResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs/{runName}/status": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "get pipeline run status", - "operationId": "getPipelineRunStatus", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1alpha1.WorkflowRunStatus" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs/{runName}/stop": { - "post": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "stop pipeline run", - "operationId": "stopPipeline", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineRunMeta" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, "/api/v1/projects": { "get": { "consumes": [ @@ -5152,6 +4258,45 @@ } } }, + "/api/v1/projects/pipelines": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "list pipelines", + "operationId": "listPipelines", + "parameters": [ + { + "type": "string", + "description": "Fuzzy search based on name or description", + "name": "query", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.ListPipelineResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, "/api/v1/projects/{projectName}": { "get": { "consumes": [ @@ -5845,6 +4990,863 @@ } } }, + "/api/v1/projects/{projectName}/pipelines": { + "post": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "create pipeline", + "operationId": "createPipeline", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.CreatePipelineRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineBase" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "get pipeline", + "operationId": "getPipeline", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.GetPipelineRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.GetPipelineResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + }, + "put": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "update pipeline", + "operationId": "updatePipeline", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.UpdatePipelineRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineBase" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + }, + "delete": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "delete pipeline", + "operationId": "deletePipeline", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineMetaResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/contexts": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "list pipeline context values", + "operationId": "listContextValues", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.ListContextValueResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + }, + "post": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "create pipeline context values", + "operationId": "createContextValue", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.CreateContextValuesRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.Context" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/contexts/{contextName}": { + "put": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "update pipeline context value", + "operationId": "updateContextValue", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.UpdateContextValuesRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline context name", + "name": "contextName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.Context" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + }, + "delete": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "delete pipeline context value", + "operationId": "deleteContextValue", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline context name", + "name": "contextName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.ContextNameResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/run": { + "post": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "run pipeline", + "operationId": "runPipeline", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.RunPipelineRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineRunMeta" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "list pipeline runs", + "operationId": "listPipelineRuns", + "parameters": [ + { + "type": "string", + "description": "query identifier of the status", + "name": "status", + "in": "query" + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.ListPipelineRunResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs/{runName}": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "get pipeline run", + "operationId": "getPipelineRun", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineRunBase" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + }, + "delete": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "delete pipeline run", + "operationId": "deletePipelineRun", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineRunMeta" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs/{runName}/log": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "get pipeline run log", + "operationId": "getPipelineRunLog", + "parameters": [ + { + "type": "string", + "description": "query by specific step name", + "name": "step", + "in": "query" + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.GetPipelineRunLogResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs/{runName}/output": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "get pipeline run output", + "operationId": "getPipelineRunOutput", + "parameters": [ + { + "type": "string", + "description": "query by specific id", + "name": "step", + "in": "query" + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.GetPipelineRunOutputResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs/{runName}/status": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "get pipeline run status", + "operationId": "getPipelineRunStatus", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1alpha1.WorkflowRunStatus" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs/{runName}/stop": { + "post": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "stop pipeline run", + "operationId": "stopPipeline", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineRunMeta" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, "/api/v1/projects/{projectName}/providers": { "get": { "consumes": [ @@ -7355,6 +7357,12 @@ "in": "path", "required": true }, + { + "type": "string", + "description": "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + "name": "dryRun", + "in": "query" + }, { "name": "body", "in": "body", @@ -8358,8 +8366,8 @@ }, "config.ClusterTargetStatus": { "required": [ - "namespace", "clusterName", + "namespace", "status", "application", "message" @@ -8648,8 +8656,8 @@ }, "model.Cluster": { "required": [ - "createTime", "updateTime", + "createTime", "name", "alias", "description", @@ -9444,12 +9452,12 @@ }, "v1.ApplicationDeployResponse": { "required": [ - "status", "note", - "createTime", - "envName", "triggerType", - "version" + "createTime", + "version", + "status", + "envName" ], "properties": { "codeInfo": { @@ -10209,13 +10217,13 @@ }, "v1.ConfigTemplateDetail": { "required": [ + "alias", + "name", "namespace", "description", "scope", "sensitive", "createTime", - "alias", - "name", "schema", "uiSchema" ], @@ -11035,11 +11043,11 @@ }, "v1.DetailAddonResponse": { "required": [ + "invisible", + "name", "version", "description", "icon", - "name", - "invisible", "schema", "uiSchema", "definitions", @@ -11119,13 +11127,13 @@ }, "v1.DetailApplicationResponse": { "required": [ - "updateTime", - "name", "alias", - "project", "description", - "createTime", "icon", + "name", + "project", + "createTime", + "updateTime", "policies", "envBindings", "resourceInfo" @@ -11182,20 +11190,20 @@ }, "v1.DetailClusterResponse": { "required": [ - "updateTime", - "name", "apiServerURL", - "kubeConfigSecret", - "createTime", "alias", - "provider", - "kubeConfig", "description", "icon", - "reason", + "kubeConfigSecret", + "name", "labels", - "status", + "reason", + "provider", "dashboardURL", + "updateTime", + "createTime", + "status", + "kubeConfig", "resourceInfo" ], "properties": { @@ -11253,14 +11261,14 @@ }, "v1.DetailComponentResponse": { "required": [ - "appPrimaryKey", + "updateTime", + "creator", + "alias", "main", "createTime", "type", "name", - "alias", - "updateTime", - "creator", + "appPrimaryKey", "definition" ], "properties": { @@ -11348,13 +11356,13 @@ }, "v1.DetailDefinitionResponse": { "required": [ + "alias", "description", "status", - "ownerAddon", - "name", - "alias", - "icon", "labels", + "name", + "icon", + "ownerAddon", "schema", "uiSchema" ], @@ -11411,15 +11419,15 @@ }, "v1.DetailPolicyResponse": { "required": [ - "name", + "envName", "alias", + "description", "creator", "properties", "createTime", - "updateTime", - "envName", + "name", "type", - "description" + "updateTime" ], "properties": { "alias": { @@ -11461,18 +11469,18 @@ }, "v1.DetailRevisionResponse": { "required": [ - "version", + "triggerType", + "note", "revisionCRName", - "workflowName", - "reason", - "updateTime", + "status", "appPrimaryKey", + "version", "envName", "createTime", - "status", + "reason", "deployUser", - "note", - "triggerType" + "workflowName", + "updateTime" ], "properties": { "appPrimaryKey": { @@ -11529,8 +11537,8 @@ }, "v1.DetailTargetResponse": { "required": [ - "updateTime", "project", + "updateTime", "name", "createTime" ], @@ -11572,11 +11580,11 @@ }, "v1.DetailUserResponse": { "required": [ - "email", - "disabled", "createTime", "lastLoginTime", "name", + "email", + "disabled", "projects", "roles" ], @@ -11674,13 +11682,13 @@ }, "v1.DetailWorkflowResponse": { "required": [ + "createTime", + "name", "alias", "enable", - "envName", - "name", - "description", "default", - "createTime", + "description", + "envName", "updateTime" ], "properties": { @@ -11913,8 +11921,8 @@ }, "v1.GetPipelineResponse": { "required": [ - "project", "name", + "project", "spec", "info" ], @@ -11941,14 +11949,27 @@ }, "v1.GetPipelineRunLogResponse": { "required": [ + "phase", + "id", + "name", + "type", "log" ], "properties": { + "id": { + "type": "string" + }, "log": { - "type": "array", - "items": { - "$ref": "#/definitions/v1.Log" - } + "type": "string" + }, + "name": { + "type": "string" + }, + "phase": { + "type": "string" + }, + "type": { + "type": "string" } } }, @@ -11960,7 +11981,7 @@ "output": { "type": "array", "items": { - "$ref": "#/definitions/v1.Output" + "$ref": "#/definitions/v1.StepOutput" } } } @@ -12526,32 +12547,6 @@ } } }, - "v1.Log": { - "required": [ - "id", - "name", - "type", - "phase", - "log" - ], - "properties": { - "id": { - "type": "string" - }, - "log": { - "type": "string" - }, - "name": { - "type": "string" - }, - "phase": { - "type": "string" - }, - "type": { - "type": "string" - } - } - }, "v1.LoginRequest": { "properties": { "code": { @@ -12729,35 +12724,6 @@ } } }, - "v1.Output": { - "required": [ - "id", - "name", - "type", - "phase", - "vars" - ], - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "phase": { - "type": "string" - }, - "type": { - "type": "string" - }, - "vars": { - "type": "object", - "additionalProperties": { - "type": "string" - } - } - } - }, "v1.PermissionBase": { "required": [ "name", @@ -12802,8 +12768,8 @@ }, "v1.PipelineBase": { "required": [ - "project", "name", + "project", "spec" ], "properties": { @@ -12891,8 +12857,8 @@ }, "v1.PipelineMetaResponse": { "required": [ - "name", - "project" + "project", + "name" ], "properties": { "alias": { @@ -12911,9 +12877,9 @@ }, "v1.PipelineRunBase": { "required": [ - "project", "pipelineRunName", "pipelineName", + "project", "record", "contextName", "spec" @@ -13331,6 +13297,52 @@ } } }, + "v1.StepOutput": { + "required": [ + "outputs", + "subStepOutputs" + ], + "properties": { + "outputs": { + "$ref": "#/definitions/v1.StepOutputBase" + }, + "subStepOutputs": { + "type": "array", + "items": { + "$ref": "#/definitions/v1.StepOutputBase" + } + } + } + }, + "v1.StepOutputBase": { + "required": [ + "phase", + "id", + "name", + "type", + "vars" + ], + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "phase": { + "type": "string" + }, + "type": { + "type": "string" + }, + "vars": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, "v1.SystemInfo": { "required": [ "platformID", @@ -13390,9 +13402,9 @@ }, "v1.SystemInfoResponse": { "required": [ + "enableCollection", "loginType", "platformID", - "enableCollection", "systemVersion" ], "properties": { @@ -14022,6 +14034,9 @@ }, "v1alpha1.WorkflowRunSpec": { "properties": { + "context": { + "type": "string" + }, "mode": { "$ref": "#/definitions/v1alpha1.WorkflowExecuteMode" }, diff --git a/e2e/addon/mock/testrepo/helm-repo/index.yaml b/e2e/addon/mock/testrepo/helm-repo/index.yaml index e496f9609..8d5bc3211 100644 --- a/e2e/addon/mock/testrepo/helm-repo/index.yaml +++ b/e2e/addon/mock/testrepo/helm-repo/index.yaml @@ -23,4 +23,15 @@ entries: annotations: system.vela: ">=1.5.0" system.kubernetes: ">=1.30.0" + vela-workflow: + - annotations: + system.vela: '>=v1.6.0-beta.1' + created: "2022-10-29T09:11:16.865230605Z" + description: vela-workflow provides the capability to run a standalone workflow + home: https://github.com/kubevela/workflow + icon: https://static.kubevela.net/images/logos/KubeVela%20-03.png + name: vela-workflow + urls: + - http://127.0.0.1:9098/helm/vela-workflow-v0.3.1.tgz + version: v0.3.1 generated: "2022-06-15T13:17:04.733573+08:00" \ No newline at end of file diff --git a/e2e/addon/mock/testrepo/helm-repo/vela-workflow-v0.3.1.tgz b/e2e/addon/mock/testrepo/helm-repo/vela-workflow-v0.3.1.tgz new file mode 100644 index 0000000000000000000000000000000000000000..94495cb851f3e9c78c5cd24a9daa3ec3289a6809 GIT binary patch literal 2006 zcmV;{2Pya;iwFP!000001MOOCZ`(E(&R6qSa0*tmTP@4i0LGiX=z1Btb=j~K?S~?0 zX^FDgNTMcEPL0n0eTSqjmSQ_gmZsRs=YuU#Vi5Z@&*)IAYi|1Hm z{;^YuD2ZggTMFhTRQ6a*=2Y}%B!fHIHMHFi>D<8=ERc(1XW)4b?mg1BWeL!TB81+* ze=imkuJjH;4xV6|kXb}qZUqzd$~@V-Heh1M;x8R1|5dc}e?UW)Fdd$^)(c#}{*RBm zzODavfqnMZG1=uQ&4?Os{hZ#2#})N|G#a+-KY#|n*8e*Ji&LISd>?fjUG8}5pa%#! zE!#U&{0VjBDuvqWl+Mx>lEU;3)HN>kG@++;oZ5?APGUEp%O3qmVyNrgJd1!>Rg5xn z;hM6mik7PQG?C2;>=P&$eBO%*@XJan>Oj zP_-XNW;r!;puzDomMVE+{Fy0W1pf>YysnCqgVsn=U=3}CB8NI3_-u-K>K;B$(`be7 zX&y@w<1CYqWq)v{eNH&9JQu=v_kP^Y6?Vg`u{c)TrnRLn|`Nl6A55G<}X>PhJ| zg{40f{-&Wjzv;#jJ%H-x;U>9kF7WX&e)s3QdW%Y`I3~)W-)e7j9%7R};G$kz<&uZn zEUrpHGpY)Q{EbJ4E+RZ2cpr}cTU+`bj?0^MmI%YL#6-H zec4@MU;pn%lq9NJh#R;58dvoH18;oP>i@mbz_b1TU0_%LU)%l~t^bYMZOy5Ad1ziM zLO;zQO=|X++U{1)G#rbn2WjmSq_rZUYtqnN3zOBK!X!!WtWR%u;+xO(&_?;`geaQPqs^p?kcK?!IF{(Q0fzO>7ijZL8mSa4lPHvr@_f zYo*U)rLU(fel=*<|Jt+uQOR%Miu&&j+V#Ib@y53P-w7aDokK&7W5N=IApV8anWApB zUu`>qm=f*l%8!cFTD&uC_>;4ZlFpD z!#O3Pi7elHZuWHbsboQ;;w5)+Ua1GtOucN7jMkk{MpOPu&nx#(Pw(VeRE-x8i=Qm~ z1<9mn{(i1sA((~w@#a#eoe>q5rEaTPo;xg_fkV!zepj(eH%5B>FNu>%wcGfZ{!P8R zJlo#Dwd23EoBjpmmVE50_#gKtm+Jp;WaIxXU?Qmbe~ZdDm*A6bA2rYG%YXa_AqO#E zllmL~NQwBIi(dIl+_wX->kZvBnL9`asPlLJy5k@u$y`hk#$Eht7&4!c2qBg%nP9UB z_8$^BLm}8`N|S&lzRD7Mwoy+Q 0 { + logs, err = getResourceLogs(ctx, p.KubeConfig, p.KubeClient, logConfig.Source.Resources, nil) + if err != nil { + return apis.GetPipelineRunLogResponse{}, err + } + } + if logConfig.Source.URL != "" { + var logsBuilder strings.Builder + readCloser, err := wfUtils.GetLogsFromURL(ctx, logConfig.Source.URL) + if err != nil { + log.Logger.Errorf("get logs from url %s failed: %v", logConfig.Source.URL, err) + return apis.GetPipelineRunLogResponse{}, bcode.ErrReadSourceLog + } + //nolint:errcheck + defer readCloser.Close() + if _, err := io.Copy(&logsBuilder, readCloser); err != nil { + log.Logger.Errorf("copy logs from url %s failed: %v", logConfig.Source.URL, err) + return apis.GetPipelineRunLogResponse{}, bcode.ErrReadSourceLog + } + logs = logsBuilder.String() + } + } + return apis.GetPipelineRunLogResponse{ + StepBase: getStepBase(pipelineRun, step), + Log: logs, + }, nil +} + +func getStepBase(run apis.PipelineRun, step string) apis.StepBase { + for _, s := range run.Status.Steps { + if s.Name == step { + return apis.StepBase{ + ID: s.ID, + Name: s.Name, + Type: s.Type, + Phase: string(s.Phase), + } + } + } + return apis.StepBase{} +} + +func getStepOutputs(step v1alpha1.StepStatus, outputsSpec map[string]v1alpha1.StepOutputs, v *value.Value) apis.StepOutputBase { + o := apis.StepOutputBase{ + StepBase: apis.StepBase{ + Name: step.Name, + ID: step.ID, + Phase: string(step.Phase), + Type: step.Type, + }, + } + values := make([]apis.OutputVar, 0) + for _, output := range outputsSpec[step.Name] { + outputValue, err := v.LookupValue(output.Name) + if err != nil { + continue + } + s, err := outputValue.String() + if err != nil { + continue + } + values = append(values, apis.OutputVar{ + Name: output.Name, + ValueFrom: output.ValueFrom, + Value: s, + }) + } + o.Values = values + return o +} + +func getStepInputs(step v1alpha1.StepStatus, inputsSpec map[string]v1alpha1.StepInputs, v *value.Value, valueFromStep map[string]string) apis.StepInputBase { + o := apis.StepInputBase{ + StepBase: apis.StepBase{ + Name: step.Name, + ID: step.ID, + Phase: string(step.Phase), + Type: step.Type, + }, + } + values := make([]apis.InputVar, 0) + for _, input := range inputsSpec[step.Name] { + outputValue, err := v.LookupValue(input.From) + if err != nil { + continue + } + s, err := outputValue.String() + if err != nil { + continue + } + values = append(values, apis.InputVar{ + Value: s, + From: input.From, + FromStep: valueFromStep[input.From], + ParameterKey: input.ParameterKey, + }) + } + o.Values = values + return o +} + +func getResourceLogs(ctx context.Context, config *rest.Config, cli client.Client, resources []wfTypes.Resource, filters []string) (string, error) { + var ( + linesOfLogKept int64 = 1000 + timeout = 10 * time.Second + errPrint = color.New(color.FgRed, color.Bold).FprintfFunc() + ) + + type PodContainer struct { + Name string + Namespace string + Container string + Label map[string]string + } + pods, err := wfUtils.GetPodListFromResources(ctx, cli, resources) + if err != nil { + log.Logger.Errorf("fail to get pod list from resources: %v", err) + return "", err + } + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + log.Logger.Errorf("fail to get clientset from kubeconfig: %v", err) + return "", err + } + podContainers := make([]PodContainer, 0) + for _, pod := range pods { + for _, container := range pod.Spec.Containers { + pc := PodContainer{ + Name: pod.Name, + Namespace: pod.Namespace, + Container: container.Name, + Label: pod.Labels, + } + podContainers = append(podContainers, pc) + } + } + + wg := sync.WaitGroup{} + logBuilder := strings.Builder{} + logMap := concurrent.NewMap() + wg.Add(len(podContainers)) + + for _, pc := range podContainers { + go func(pc PodContainer) { + defer wg.Done() + ctxQuery, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + podLogOpts := corev1.PodLogOptions{ + Container: pc.Container, + Follow: false, + TailLines: &linesOfLogKept, + } + req := clientSet.CoreV1().Pods(pc.Namespace).GetLogs(pc.Name, &podLogOpts) + podLogs, err := req.Stream(ctxQuery) + if err != nil { + log.Logger.Errorf("fail to get pod logs: %v", err) + return + } + defer func() { + _ = podLogs.Close() + }() + podColor, containerColor := determineColor(pc.Name) + buf := new(bytes.Buffer) + p := podColor.SprintfFunc() + c := containerColor.SprintfFunc() + fmt.Fprintf(buf, "%s %s\n", p("› %s", pc.Name), c("%s", pc.Container)) + var readErr error + r := bufio.NewReader(podLogs) + for { + s, err := r.ReadString('\n') + if err != nil { + if !errors.Is(err, io.EOF) { + readErr = err + } + break + } + if len(filters) != 0 { + for _, f := range filters { + if strings.Contains(s, f) { + buf.WriteString(s) + } + } + } + } + if readErr != nil { + errPrint(buf, "error in copy information from APIServer to buffer: %s", err.Error()) + log.Logger.Errorf("fail to copy pod logs: %v", err) + } + logMap.Store(fmt.Sprintf("%s/%s", pc.Name, pc.Container), buf.String()) + }(pc) + } + wg.Wait() + + order := make([]string, 0) + sort.Strings(order) + logMap.Range(func(key, value any) bool { + order = append(order, key.(string)) + return true + }) + for _, key := range order { + val, ok := logMap.Load(key) + if ok { + logBuilder.WriteString(fmt.Sprintf("%s", val)) + } + } + return logBuilder.String(), nil +} + // RunPipeline will run a pipeline -func (p pipelineServiceImpl) RunPipeline(ctx context.Context, pipeline apis.PipelineBase, req apis.RunPipelineRequest) error { - // todo get context and set to workflowRun +func (p pipelineServiceImpl) RunPipeline(ctx context.Context, pipeline apis.PipelineBase, req apis.RunPipelineRequest) (*apis.PipelineRun, error) { + if err := checkRunMode(&req.Mode); err != nil { + return nil, err + } + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) run := v1alpha1.WorkflowRun{} version := utils.GenerateVersion("") name := fmt.Sprintf("%s-%s", pipeline.Name, version) run.Name = name - run.Namespace = fmt.Sprintf("%s-project", pipeline.Project) - run.Spec.WorkflowRef = pipeline.Name + run.Namespace = project.GetNamespace() + run.Spec.WorkflowSpec = &pipeline.Spec run.Spec.Mode = &req.Mode - return p.KubeClient.Create(ctx, &run) + run.SetLabels(map[string]string{ + labelPipeline: pipeline.Name, + model.LabelSourceOfTruth: model.FromUX, + }) + + // process the context + if req.ContextName != "" { + ppContext, err := p.ContextService.GetContext(ctx, pipeline.Project.Name, pipeline.Name, req.ContextName) + if err != nil { + return nil, err + } + contextData := make(map[string]interface{}) + for _, pair := range ppContext.Values { + contextData[pair.Key] = pair.Value + } + run.Labels[labelContext] = req.ContextName + run.Spec.Context = util.Object2RawExtension(contextData) + } + + if err := p.KubeClient.Create(ctx, &run); err != nil { + return nil, err + } + + return p.PipelineRunService.GetPipelineRun(ctx, apis.PipelineRunMeta{ + PipelineName: pipeline.Name, + Project: apis.NameAlias{Name: project.Name}, + PipelineRunName: name, + }) +} + +// getPipelineInfo returns the pipeline statistic info +// return error can be nil if pipeline hasn't been run +func (p pipelineServiceImpl) getPipelineInfo(ctx context.Context, wf *model.Pipeline, namespace string) (*apis.PipelineInfo, error) { + var runs v1alpha1.WorkflowRunList + err := p.KubeClient.List(context.Background(), &runs, client.InNamespace(namespace), client.MatchingLabels{labelPipeline: wf.Name}) + if err != nil { + if meta.IsNoMatchError(err) { + weekStat := make([]apis.RunStatInfo, 0) + for i := 0; i < 7; i++ { + weekStat = append(weekStat, apis.RunStatInfo{}) + } + return &apis.PipelineInfo{ + LastRun: nil, + RunStat: apis.RunStat{ + ActiveNum: 0, + Total: apis.RunStatInfo{}, + Week: weekStat, + }, + }, nil + } + return nil, err + } + run := getLastRun(runs.Items) + runStat := getRunStat(runs.Items) + pi := &apis.PipelineInfo{ + RunStat: runStat, + } + if run != nil { + projectName := wf.Project + project, err := p.ProjectService.GetProject(ctx, projectName) + if err != nil { + return nil, err + } + pi.LastRun, err = workflowRun2PipelineRun(*run, project, p.ContextService) + if err != nil { + return nil, err + } + } + return pi, nil +} + +func getRunStat(runs []v1alpha1.WorkflowRun) apis.RunStat { + today := time.Now().Unix() + isActive := func(run v1alpha1.WorkflowRun) bool { + return !run.Status.Finished && !run.Status.Terminated && run.Status.Suspend + } + isSuccess := func(run v1alpha1.WorkflowRun) bool { + return run.Status.Phase == v1alpha1.WorkflowStateSucceeded + } + // returned int x means the (x+1)/7 days of the week, valid number is 0-6 + inThisWeek := func(run v1alpha1.WorkflowRun) (int, bool) { + startTime := run.Status.StartTime.Time.Unix() + // one week, note this is not week of natual, but week of unix timestamp + if today-startTime < 604800 { + return 6 - int((today-startTime)/86400), true + } + return -1, false + } + var ( + act int + success int + fail int + week = make([]apis.RunStatInfo, 7) + ) + + for _, run := range runs { + // total = success + fail + active + day, inWeek := inThisWeek(run) + if inWeek { + week[day].Total++ + } + if isActive(run) { + act++ + } else { + if isSuccess(run) { + if inWeek { + week[day].Success++ + } + success++ + } else { + fail++ + if inWeek { + week[day].Fail++ + } + } + } + + } + return apis.RunStat{ + ActiveNum: act, + Total: apis.RunStatInfo{ + Total: len(runs), + Success: success, + Fail: fail, + }, + Week: week, + } +} + +func getLastRun(runs []v1alpha1.WorkflowRun) *v1alpha1.WorkflowRun { + if len(runs) == 0 { + return nil + } + last := runs[0] + lastStartTime := last.Status.StartTime.Time + for _, wfr := range runs { + wfr.Status.StartTime.After(lastStartTime) + last = wfr + lastStartTime = wfr.Status.StartTime.Time + } + // We don't need managed fields, save some bandwidth + last.ManagedFields = nil + return &last } // GetPipelineRun will get a pipeline run -func (p pipelineRunServiceImpl) GetPipelineRun(ctx context.Context, meta apis.PipelineRunMeta) (apis.PipelineRun, error) { - namespacedName := client.ObjectKey{Name: meta.PipelineName, Namespace: nsForProj(meta.Project)} +func (p pipelineRunServiceImpl) GetPipelineRun(ctx context.Context, meta apis.PipelineRunMeta) (*apis.PipelineRun, error) { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) + namespacedName := client.ObjectKey{Name: meta.PipelineRunName, Namespace: project.GetNamespace()} run := v1alpha1.WorkflowRun{} if err := p.KubeClient.Get(ctx, namespacedName, &run); err != nil { - return apis.PipelineRun{}, err + return nil, err } - return workflowRun2PipelineRun(run), nil + if run.Labels != nil && run.Labels[labelPipeline] != "" { + pipeline := &model.Pipeline{ + Name: run.Labels[labelPipeline], + Project: project.Name, + } + if err := p.Store.Get(ctx, pipeline); err != nil { + log.Logger.Errorf("failed to load the workflow %s", err.Error()) + if errors.Is(err, datastore.ErrRecordNotExist) { + return nil, bcode.ErrPipelineNotExist + } + return nil, err + } + run.Spec.WorkflowSpec = &pipeline.Spec + + } + return workflowRun2PipelineRun(run, project, p.ContextService) } // ListPipelineRuns will list all pipeline runs func (p pipelineRunServiceImpl) ListPipelineRuns(ctx context.Context, base apis.PipelineBase) (apis.ListPipelineRunResponse, error) { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) wfrs := v1alpha1.WorkflowRunList{} - if err := p.KubeClient.List(ctx, &wfrs, client.InNamespace(nsForProj(base.Project))); err != nil { + if err := p.KubeClient.List(ctx, &wfrs, client.InNamespace(project.GetNamespace()), client.MatchingLabels{labelPipeline: base.Name}); err != nil { return apis.ListPipelineRunResponse{}, err } - res := apis.ListPipelineRunResponse{} - for _, wfr := range wfrs.Items { - if wfr.Spec.WorkflowRef == base.Name { - res.Runs = append(res.Runs, p.workflowRun2runBriefing(ctx, wfr)) - } + res := apis.ListPipelineRunResponse{ + Runs: make([]apis.PipelineRunBriefing, 0), } + for _, wfr := range wfrs.Items { + res.Runs = append(res.Runs, p.workflowRun2runBriefing(ctx, wfr, project)) + } + res.Total = int64(len(res.Runs)) return res, nil } // DeletePipelineRun will delete a pipeline run func (p pipelineRunServiceImpl) DeletePipelineRun(ctx context.Context, meta apis.PipelineRunMeta) error { - namespacedName := client.ObjectKey{Name: meta.PipelineName, Namespace: nsForProj(meta.Project)} - run := v1alpha1.WorkflowRun{} - if err := p.KubeClient.Get(ctx, namespacedName, &run); err != nil { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) + run := v1alpha1.WorkflowRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: meta.PipelineRunName, + Namespace: project.GetNamespace(), + }, + } + err := p.KubeClient.Delete(ctx, &run) + return client.IgnoreNotFound(err) +} + +// CleanPipelineRuns will clean all pipeline runs, it equals to call ListPipelineRuns and multiple DeletePipelineRun +func (p pipelineRunServiceImpl) CleanPipelineRuns(ctx context.Context, base apis.PipelineBase) error { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) + wfrs := v1alpha1.WorkflowRunList{} + if err := p.KubeClient.List(ctx, &wfrs, client.InNamespace(project.GetNamespace()), client.MatchingLabels{labelPipeline: base.Name}); err != nil { return err } - return p.KubeClient.Delete(ctx, &run) + for _, wfr := range wfrs.Items { + if err := p.KubeClient.Delete(ctx, wfr.DeepCopy()); err != nil { + return client.IgnoreNotFound(err) + } + } + return nil +} + +// InitContext will init pipeline context record +func (c contextServiceImpl) InitContext(ctx context.Context, projectName, pipelineName string) (*model.PipelineContext, error) { + modelCtx := model.PipelineContext{ + ProjectName: projectName, + PipelineName: pipelineName, + } + if err := c.Store.Get(ctx, &modelCtx); err == nil { + return nil, bcode.ErrContextAlreadyExist + } + modelCtx.Contexts = make(map[string][]model.Value) + if err := c.Store.Add(ctx, &modelCtx); err != nil { + return nil, err + } + return &modelCtx, nil } // GetContext will get a context @@ -263,29 +962,36 @@ func (c contextServiceImpl) GetContext(ctx context.Context, projectName, pipelin } vals, ok := modelCtx.Contexts[name] if !ok { - return nil, errors.New("context not found") + return nil, bcode.ErrContextNotFound } return &apis.Context{Name: name, Values: vals}, nil } // CreateContext will create a context func (c contextServiceImpl) CreateContext(ctx context.Context, projectName, pipelineName string, context apis.Context) (*model.PipelineContext, error) { - modelCtx := model.PipelineContext{ + modelCtx := &model.PipelineContext{ ProjectName: projectName, PipelineName: pipelineName, } - if err := c.Store.Get(ctx, &modelCtx); err != nil { - return nil, err + if err := c.Store.Get(ctx, modelCtx); err != nil { + if errors.Is(err, datastore.ErrRecordNotExist) { + modelCtx, err = c.InitContext(ctx, projectName, pipelineName) + if err != nil { + return nil, err + } + } else { + return nil, err + } } if _, ok := modelCtx.Contexts[context.Name]; ok { log.Logger.Errorf("context %s already exists", context.Name) return nil, bcode.ErrContextAlreadyExist } modelCtx.Contexts[context.Name] = context.Values - if err := c.Store.Put(ctx, &modelCtx); err != nil { + if err := c.Store.Put(ctx, modelCtx); err != nil { return nil, err } - return &modelCtx, nil + return modelCtx, nil } // UpdateContext will update a context @@ -311,6 +1017,12 @@ func (c contextServiceImpl) ListContexts(ctx context.Context, projectName, pipel PipelineName: pipelineName, } if err := c.Store.Get(ctx, &modelCtx); err != nil { + if errors.Is(err, datastore.ErrRecordNotExist) { + return &apis.ListContextValueResponse{ + Total: 0, + Contexts: make(map[string][]model.Value), + }, nil + } return nil, err } return &apis.ListContextValueResponse{ @@ -329,99 +1041,164 @@ func (c contextServiceImpl) DeleteContext(ctx context.Context, projectName, pipe return err } delete(modelCtx.Contexts, name) - if err := c.Store.Put(ctx, &modelCtx); err != nil { - return err + return c.Store.Put(ctx, &modelCtx) +} + +// DeleteAllContexts will delete all contexts of a pipeline +func (c contextServiceImpl) DeleteAllContexts(ctx context.Context, projectName, pipelineName string) error { + modelCtx := model.PipelineContext{ + ProjectName: projectName, + PipelineName: pipelineName, } - return nil + return c.Store.Delete(ctx, &modelCtx) } -func nsForProj(proj string) string { - return fmt.Sprintf("project-%s", proj) -} - -func getWfDescription(wf v1alpha1.Workflow) string { - if wf.Labels == nil { - return "" - } - return wf.Labels[labelDescription] -} - -func getWfAlias(wf v1alpha1.Workflow) string { - if wf.Labels == nil { - return "" - } - return wf.Labels[labelAlias] -} - -func fuzzyMatch(wf v1alpha1.Workflow, q string) bool { +func fuzzyMatch(wf *model.Pipeline, q string) bool { if strings.Contains(wf.Name, q) { return true } - if strings.Contains(getWfAlias(wf), q) { + if strings.Contains(wf.Alias, q) { return true } - if strings.Contains(getWfDescription(wf), q) { + if strings.Contains(wf.Description, q) { return true } return false } -func workflow2PipelineBase(wf v1alpha1.Workflow) *apis.PipelineBase { - project := strings.TrimRight(wf.Namespace, "-project") +func pipeline2PipelineBase(wf *model.Pipeline, project model.Project) *apis.PipelineBase { return &apis.PipelineBase{ PipelineMeta: apis.PipelineMeta{ - Name: wf.Name, - Project: project, - Description: getWfDescription(wf), - Alias: getWfAlias(wf), + Name: wf.Name, + Project: apis.NameAlias{ + Name: project.Name, + Alias: project.Alias, + }, + Description: wf.Description, + Alias: wf.Alias, + CreateTime: wf.CreateTime, }, - Spec: wf.WorkflowSpec, + Spec: wf.Spec, } } -func workflowRun2PipelineRun(run v1alpha1.WorkflowRun) apis.PipelineRun { - return apis.PipelineRun{ +func workflowRun2PipelineRun(run v1alpha1.WorkflowRun, project *model.Project, ctxService ContextService) (*apis.PipelineRun, error) { + mergeSteps(&run) + pipelineName := run.Labels[labelPipeline] + pipelineRun := &apis.PipelineRun{ PipelineRunBase: apis.PipelineRunBase{ PipelineRunMeta: apis.PipelineRunMeta{ - PipelineName: run.Spec.WorkflowRef, - Project: strings.TrimRight(run.Namespace, "-project"), + PipelineName: pipelineName, + Project: apis.NameAlias{ + Name: project.Name, + Alias: project.Alias, + }, PipelineRunName: run.Name, }, + Spec: run.Spec, }, Status: run.Status, } + if labels := run.GetLabels(); labels != nil { + ctxName := labels[labelContext] + if ctxName != "" { + ctx, err := ctxService.GetContext(context.Background(), project.Name, pipelineName, ctxName) + if err != nil && !errors.Is(err, datastore.ErrRecordNotExist) { + return nil, err + } + pipelineRun.PipelineRunBase.ContextName = ctxName + pipelineRun.PipelineRunBase.ContextValues = ctx.Values + } + } + return pipelineRun, nil } -func (p pipelineRunServiceImpl) workflowRun2runBriefing(ctx context.Context, run v1alpha1.WorkflowRun) apis.PipelineRunBriefing { - contextName := run.Labels[labelContextName] - project := strings.TrimRight(run.Namespace, "-project") - apiContext, err := p.ContextService.GetContext(ctx, project, run.Spec.WorkflowRef, contextName) - if err != nil { - log.Logger.Warnf("failed to get pipeline run context %s/%s/%s: %v", project, run.Spec.WorkflowRef, contextName, err) - apiContext = nil +func mergeSteps(run *v1alpha1.WorkflowRun) { + if run.Spec.WorkflowSpec == nil { + return + } + if run.Status.Steps == nil { + run.Status.Steps = make([]v1alpha1.WorkflowStepStatus, 0) + } + var stepStatus = make(map[string]*v1alpha1.WorkflowStepStatus, len(run.Status.Steps)) + for i, step := range run.Status.Steps { + stepStatus[step.Name] = &run.Status.Steps[i] + } + for _, step := range run.Spec.WorkflowSpec.Steps { + if stepStatusCache, exist := stepStatus[step.Name]; !exist { + var subSteps []v1alpha1.StepStatus + for _, subStep := range step.SubSteps { + subSteps = append(subSteps, v1alpha1.StepStatus{ + Name: subStep.Name, + Type: subStep.Type, + Phase: v1alpha1.WorkflowStepPhasePending, + }) + } + run.Status.Steps = append(run.Status.Steps, v1alpha1.WorkflowStepStatus{ + StepStatus: v1alpha1.StepStatus{ + Name: step.Name, + Type: step.Type, + Phase: v1alpha1.WorkflowStepPhasePending, + }, + SubStepsStatus: subSteps, + }) + } else if len(step.SubSteps) > len(stepStatusCache.SubStepsStatus) { + var subStepStatus = make(map[string]v1alpha1.StepStatus, len(stepStatusCache.SubStepsStatus)) + for i, step := range stepStatusCache.SubStepsStatus { + subStepStatus[step.Name] = stepStatusCache.SubStepsStatus[i] + } + for _, subStep := range step.SubSteps { + if _, exist := subStepStatus[subStep.Name]; !exist { + stepStatusCache.SubStepsStatus = append(stepStatusCache.SubStepsStatus, v1alpha1.StepStatus{ + Name: subStep.Name, + Type: subStep.Type, + Phase: v1alpha1.WorkflowStepPhasePending, + }) + } + } + } + } +} + +func (p pipelineRunServiceImpl) workflowRun2runBriefing(ctx context.Context, run v1alpha1.WorkflowRun, project *model.Project) apis.PipelineRunBriefing { + var ( + apiContext *apis.Context + err error + ) + pipelineName := run.Labels[labelPipeline] + if contextName, ok := run.Labels[labelContext]; ok { + apiContext, err = p.ContextService.GetContext(ctx, project.Name, pipelineName, contextName) + if err != nil { + log.Logger.Warnf("failed to get pipeline run context %s/%s/%s: %v", project, pipelineName, contextName, err) + apiContext = nil + } } - return apis.PipelineRunBriefing{ + briefing := apis.PipelineRunBriefing{ PipelineRunName: run.Name, Finished: run.Status.Finished, Phase: run.Status.Phase, Message: run.Status.Message, StartTime: run.Status.StartTime, EndTime: run.Status.EndTime, - ContextName: apiContext.Name, - ContextValues: apiContext.Values, } + if apiContext != nil { + briefing.ContextName = apiContext.Name + briefing.ContextValues = apiContext.Values + } + return briefing } -func (p pipelineRunServiceImpl) checkRecordRunning(ctx context.Context, pipelineRun apis.PipelineRunBase) (*v1alpha1.WorkflowRun, error) { +func (p pipelineRunServiceImpl) checkRunNotFinished(ctx context.Context, pipelineRun apis.PipelineRunBase) (*v1alpha1.WorkflowRun, error) { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) run := v1alpha1.WorkflowRun{} if err := p.KubeClient.Get(ctx, types.NamespacedName{ - Namespace: nsForProj(pipelineRun.Project), + Namespace: project.GetNamespace(), Name: pipelineRun.PipelineRunName, }, &run); err != nil { return nil, err } - if !run.Status.Suspend && !run.Status.Terminated && !run.Status.Finished { - return nil, fmt.Errorf("workflow is still running, can not operate a running workflow") + if run.Status.Terminated || run.Status.Finished { + return nil, bcode.ErrPipelineRunFinished } return &run, nil } @@ -461,3 +1238,59 @@ func (p pipelineRunServiceImpl) terminatePipelineRun(ctx context.Context, run *v return nil } + +func checkPipelineSpec(spec v1alpha1.WorkflowSpec) error { + if len(spec.Steps) == 0 { + return bcode.ErrNoSteps + } + return nil +} + +func checkRunMode(mode *v1alpha1.WorkflowExecuteMode) error { + if mode.Steps == "" { + mode.Steps = v1alpha1.WorkflowModeStep + } + if mode.SubSteps == "" { + mode.SubSteps = v1alpha1.WorkflowModeDAG + } + if mode.Steps != v1alpha1.WorkflowModeStep && mode.Steps != v1alpha1.WorkflowModeDAG && + mode.SubSteps != v1alpha1.WorkflowModeStep && mode.SubSteps != v1alpha1.WorkflowModeDAG { + return bcode.ErrWrongMode + } + return nil +} + +// NewTestPipelineService create the pipeline service instance for testing +func NewTestPipelineService(ds datastore.DataStore, c client.Client, cfg *rest.Config) PipelineService { + projectService := NewTestProjectService(ds, c) + contextService := NewTestContextService(ds) + ppRunService := NewTestPipelineRunService(ds, c, cfg) + pipelineService := &pipelineServiceImpl{ + ProjectService: projectService, + ContextService: contextService, + KubeClient: c, + KubeConfig: cfg, + PipelineRunService: ppRunService, + Store: ds, + } + return pipelineService +} + +// NewTestPipelineRunService create the pipeline run service instance for testing +func NewTestPipelineRunService(ds datastore.DataStore, c client.Client, cfg *rest.Config) PipelineRunService { + contextService := NewTestContextService(ds) + projectService := NewTestProjectService(ds, c) + return &pipelineRunServiceImpl{ + KubeClient: c, + KubeConfig: cfg, + ContextService: contextService, + ProjectService: projectService, + } +} + +// NewTestContextService create the context service instance for testing +func NewTestContextService(ds datastore.DataStore) ContextService { + return &contextServiceImpl{ + Store: ds, + } +} diff --git a/pkg/apiserver/domain/service/pipeline_test.go b/pkg/apiserver/domain/service/pipeline_test.go new file mode 100644 index 000000000..5214a6254 --- /dev/null +++ b/pkg/apiserver/domain/service/pipeline_test.go @@ -0,0 +1,143 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + + "github.com/kubevela/workflow/api/v1alpha1" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/oam-dev/kubevela/pkg/apiserver/domain/model" + "github.com/oam-dev/kubevela/pkg/apiserver/infrastructure/datastore" + apisv1 "github.com/oam-dev/kubevela/pkg/apiserver/interfaces/api/dto/v1" + "github.com/oam-dev/kubevela/pkg/oam/util" +) + +var ( + // defaultNamespace = "project-default-ns1-test" + pipelineService *pipelineServiceImpl + pipelineRunService *pipelineRunServiceImpl + userService *userServiceImpl + contextService *contextServiceImpl + projectService *projectServiceImpl + ctx context.Context + + pipelineName = "test-pipeline" + projectName = "test-project" +) +var _ = Describe("Test pipeline service functions", func() { + It("Init services and project", func() { + ds, err := NewDatastore(datastore.Config{Type: "kubeapi", Database: "pipeline-test-kubevela"}) + Expect(ds).ToNot(BeNil()) + Expect(err).Should(BeNil()) + Expect(err).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + pipelineService = NewTestPipelineService(ds, k8sClient, cfg).(*pipelineServiceImpl) + pipelineRunService = pipelineService.PipelineRunService.(*pipelineRunServiceImpl) + contextService = pipelineService.ContextService.(*contextServiceImpl) + projectService = pipelineService.ProjectService.(*projectServiceImpl) + userService = &userServiceImpl{Store: ds, K8sClient: k8sClient} + + ctx = context.WithValue(context.TODO(), &apisv1.CtxKeyUser, "admin") + err = userService.Init(context.TODO()) + Expect(err).Should(BeNil()) + _, err = projectService.CreateProject(ctx, apisv1.CreateProjectRequest{ + Name: projectName, + Owner: "admin", + }) + Expect(err).Should(BeNil()) + projModel, err := projectService.GetProject(context.TODO(), projectName) + Expect(err).Should(BeNil()) + ctx = context.WithValue(ctx, &apisv1.CtxKeyProject, projModel) + }) + + It("Test create pipeline", func() { + rawProps := []byte(`{"url":"https://api.github.com/repos/kubevela/kubevela"}`) + testPipelineSteps := []v1alpha1.WorkflowStep{ + { + SubSteps: []v1alpha1.WorkflowStepBase{ + { + Name: "request", + Type: "request", + Outputs: v1alpha1.StepOutputs{ + { + ValueFrom: "import \"strconv\"\n\"Current star count: \" + strconv.FormatInt(response[\"stargazers_count\"], 10)\n", + Name: "stars", + }, + }, + Properties: &runtime.RawExtension{ + Raw: rawProps, + }, + }, + }, + WorkflowStepBase: v1alpha1.WorkflowStepBase{ + Name: "step-group", + Type: "step-group", + }, + }, + } + + By("create pipeline with sub-steps") + pipeline, err := pipelineService.CreatePipeline(ctx, apisv1.CreatePipelineRequest{ + Name: pipelineName, + Spec: v1alpha1.WorkflowSpec{ + Steps: testPipelineSteps, + }, + }) + Expect(err).Should(BeNil()) + Expect(pipeline.Name).Should(Equal(pipelineName)) + Expect(pipeline.Spec.Steps[0].Name).Should(Equal("step-group")) + }) + + It("list pipeline", func() { + pipelines, err := pipelineService.ListPipelines(ctx, apisv1.ListPipelineRequest{ + Detailed: true, + }) + Expect(err).Should(BeNil()) + Expect(pipelines).ShouldNot(BeNil()) + Expect(pipelines.Total).Should(Equal(1)) + Expect(len(pipelines.Pipelines)).Should(Equal(1)) + Expect(pipelines.Pipelines[0].Info).ShouldNot(BeNil()) + }) + + It("get pipeline contexts", func() { + By("no context") + contexts, err := contextService.ListContexts(ctx, projectName, pipelineName) + Expect(err).Should(BeNil()) + Expect(contexts.Total).Should(Equal(0)) + Expect(len(contexts.Contexts)).Should(Equal(0)) + + By("create context") + contextName := "test-context" + contextKey := "test-key" + contextVal := "test-val" + ppCtx := apisv1.Context{ + Name: contextName, + Values: []model.Value{ + { + Key: contextKey, + Value: contextVal, + }, + }, + } + context, err := contextService.CreateContext(ctx, projectName, pipelineName, ppCtx) + Expect(err).Should(BeNil()) + Expect(len(context.Contexts)).Should(Equal(1)) + }) +}) diff --git a/pkg/apiserver/domain/service/project.go b/pkg/apiserver/domain/service/project.go index 73227e896..94fe75db8 100644 --- a/pkg/apiserver/domain/service/project.go +++ b/pkg/apiserver/domain/service/project.go @@ -22,6 +22,7 @@ import ( "fmt" terraformapi "github.com/oam-dev/terraform-controller/api/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/oam-dev/kubevela/apis/types" @@ -31,6 +32,7 @@ import ( "github.com/oam-dev/kubevela/pkg/apiserver/utils/bcode" "github.com/oam-dev/kubevela/pkg/apiserver/utils/log" "github.com/oam-dev/kubevela/pkg/multicluster" + "github.com/oam-dev/kubevela/pkg/utils" ) // ProjectService project manage service. @@ -159,6 +161,13 @@ func (p *projectServiceImpl) GetProject(ctx context.Context, projectName string) } return nil, err } + if _, err := utils.GetNamespace(ctx, p.K8sClient, project.GetNamespace()); err != nil { + if apierrors.IsNotFound(err) { + if err := utils.CreateNamespace(ctx, p.K8sClient, projectName); err != nil && !apierrors.IsAlreadyExists(err) { + return nil, bcode.ErrProjectNamespaceFail + } + } + } return project, nil } @@ -319,11 +328,16 @@ func (p *projectServiceImpl) CreateProject(ctx context.Context, req apisv1.Creat } } + if err := utils.CreateNamespace(ctx, p.K8sClient, req.Name); err != nil && !apierrors.IsAlreadyExists(err) { + return nil, bcode.ErrProjectNamespaceFail + } + newProject := &model.Project{ Name: req.Name, Description: req.Description, Alias: req.Alias, Owner: owner, + Namespace: req.Name, } if err := p.Store.Add(ctx, newProject); err != nil { @@ -526,6 +540,7 @@ func ConvertProjectModel2Base(project *model.Project, owner *model.User) *apisv1 CreateTime: project.CreateTime, UpdateTime: project.UpdateTime, Owner: apisv1.NameAlias{Name: project.Owner}, + Namespace: project.GetNamespace(), } if owner != nil && owner.Name == project.Owner { base.Owner = apisv1.NameAlias{Name: owner.Name, Alias: owner.Alias} diff --git a/pkg/apiserver/domain/service/rbac.go b/pkg/apiserver/domain/service/rbac.go index eb626f3e3..22c909773 100644 --- a/pkg/apiserver/domain/service/rbac.go +++ b/pkg/apiserver/domain/service/rbac.go @@ -59,6 +59,7 @@ var defaultProjectPermissionTemplate = []*model.PermissionTemplate{ "project:{projectName}/permission:*", "project:{projectName}/environment:*", "project:{projectName}/application:*/*", + "project:{projectName}/pipeline:*/*", }, Actions: []string{"detail", "list"}, Effect: "Allow", @@ -96,6 +97,16 @@ var defaultProjectPermissionTemplate = []*model.PermissionTemplate{ Effect: "Allow", Scope: "project", }, + { + Name: "pipeline-management", + Alias: "Pipeline Management", + Resources: []string{ + "project:{projectName}/pipeline:*", + }, + Actions: []string{"*"}, + Effect: "Allow", + Scope: "project", + }, } var defaultPlatformPermission = []*model.PermissionTemplate{ @@ -234,6 +245,17 @@ var ResourceMaps = map[string]resourceMetadata{ pathName: "configName", }, "provider": {}, + "pipeline": { + pathName: "pipelineName", + subResources: map[string]resourceMetadata{ + "context": { + pathName: "contextName", + }, + "pipelineRun": { + pathName: "pipelineRunName", + }, + }, + }, }, pathName: "projectName", }, @@ -866,7 +888,7 @@ func (p *rbacServiceImpl) InitDefaultRoleAndUsersForProject(ctx context.Context, }, &model.Role{ Name: "project-admin", Alias: "Project Admin", - Permissions: []string{"project-view", "app-management", "env-management", "role-management", "configuration-read"}, + Permissions: []string{"project-view", "app-management", "env-management", "role-management", "pipeline-management", "configuration-read"}, Project: project.Name, }, &model.Role{ Name: "project-viewer", diff --git a/pkg/apiserver/domain/service/rbac_test.go b/pkg/apiserver/domain/service/rbac_test.go index 11c442c51..3cc73d0ca 100644 --- a/pkg/apiserver/domain/service/rbac_test.go +++ b/pkg/apiserver/domain/service/rbac_test.go @@ -198,7 +198,7 @@ var _ = Describe("Test rbac service", func() { policies, err := rbacService.ListPermissions(context.TODO(), "init-test") Expect(err).Should(BeNil()) - Expect(len(policies)).Should(BeEquivalentTo(int64(5))) + Expect(len(policies)).Should(BeEquivalentTo(int64(6))) }) It("Test UpdatePermission", func() { diff --git a/pkg/apiserver/domain/service/suite_test.go b/pkg/apiserver/domain/service/suite_test.go index 850a910b8..39bcb808b 100644 --- a/pkg/apiserver/domain/service/suite_test.go +++ b/pkg/apiserver/domain/service/suite_test.go @@ -25,6 +25,7 @@ import ( "testing" "time" + "github.com/kubevela/workflow/api/v1alpha1" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -74,7 +75,10 @@ var _ = BeforeSuite(func(done Done) { By("new kube client") cfg.Timeout = time.Minute * 2 - k8sClient, err = client.New(cfg, client.Options{Scheme: common.Scheme}) + scheme := common.Scheme + err = v1alpha1.AddToScheme(scheme) + Expect(err).ShouldNot(HaveOccurred()) + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) Expect(err).Should(BeNil()) Expect(k8sClient).ToNot(BeNil()) By("new kube client success") diff --git a/pkg/apiserver/domain/service/workflow_test.go b/pkg/apiserver/domain/service/workflow_test.go index 7b3f5f265..cee26d258 100644 --- a/pkg/apiserver/domain/service/workflow_test.go +++ b/pkg/apiserver/domain/service/workflow_test.go @@ -58,7 +58,7 @@ var _ = Describe("Test workflow service functions", func() { Expect(ds).ToNot(BeNil()) Expect(err).Should(BeNil()) rbacService := &rbacServiceImpl{Store: ds} - projectService = &projectServiceImpl{Store: ds, RbacService: rbacService} + projectService = &projectServiceImpl{Store: ds, RbacService: rbacService, K8sClient: k8sClient} envService = &envServiceImpl{Store: ds, KubeClient: k8sClient, ProjectService: projectService} envBinding = &envBindingServiceImpl{ Store: ds, diff --git a/pkg/apiserver/infrastructure/clients/kubeclient.go b/pkg/apiserver/infrastructure/clients/kubeclient.go index 436773a31..a92e03437 100644 --- a/pkg/apiserver/infrastructure/clients/kubeclient.go +++ b/pkg/apiserver/infrastructure/clients/kubeclient.go @@ -20,13 +20,13 @@ import ( "fmt" pkgmulticluster "github.com/kubevela/pkg/multicluster" + "github.com/kubevela/workflow/api/v1alpha1" + "github.com/kubevela/workflow/pkg/cue/packages" "k8s.io/client-go/discovery" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" - "github.com/kubevela/workflow/pkg/cue/packages" - apiConfig "github.com/oam-dev/kubevela/pkg/apiserver/config" "github.com/oam-dev/kubevela/pkg/auth" "github.com/oam-dev/kubevela/pkg/oam/discoverymapper" @@ -73,11 +73,13 @@ func GetKubeClient() (client.Client, error) { if kubeConfig == nil { return nil, fmt.Errorf("please call SetKubeConfig first") } - var err error - kubeClient, err = pkgmulticluster.NewClient(kubeConfig, pkgmulticluster.ClientOptions{ + err := v1alpha1.AddToScheme(common.Scheme) + if err != nil { + return nil, err + } + return pkgmulticluster.NewClient(kubeConfig, pkgmulticluster.ClientOptions{ Options: client.Options{Scheme: common.Scheme}, }) - return kubeClient, err } // GetKubeConfig create/get kube runtime config diff --git a/pkg/apiserver/interfaces/api/dto/v1/types.go b/pkg/apiserver/interfaces/api/dto/v1/types.go index a8a8ad298..0138b4c6d 100644 --- a/pkg/apiserver/interfaces/api/dto/v1/types.go +++ b/pkg/apiserver/interfaces/api/dto/v1/types.go @@ -50,12 +50,14 @@ var ( CtxKeyApplicationComponent = "component" // CtxKeyUser request context key of user CtxKeyUser = "user" + // CtxKeyProject request context key of project + CtxKeyProject = "project" // CtxKeyToken request context key of request token CtxKeyToken = "token" // CtxKeyPipeline request context key of pipeline CtxKeyPipeline = "pipeline" - // CtxKeyPipelineContex request context key of pipeline context - CtxKeyPipelineContex = "pipeline-context" + // CtxKeyPipelineContext request context key of pipeline context + CtxKeyPipelineContext = "pipeline-context" // CtxKeyPipelineRun request context key of pipeline run CtxKeyPipelineRun = "pipeline-run" ) @@ -791,6 +793,7 @@ type ProjectBase struct { CreateTime time.Time `json:"createTime"` UpdateTime time.Time `json:"updateTime"` Owner NameAlias `json:"owner,omitempty"` + Namespace string `json:"-"` } // CreateProjectRequest create project request body @@ -1551,10 +1554,11 @@ type ListConfigDistributionResponse struct { // PipelineMeta is metadata of pipeline type PipelineMeta struct { - Name string `json:"name" validate:"checkname"` - Alias string `json:"alias" validate:"checkalias" optional:"true"` - Project string `json:"project"` - Description string `json:"description" optional:"true"` + Name string `json:"name"` + Alias string `json:"alias"` + Project NameAlias `json:"project"` + Description string `json:"description"` + CreateTime time.Time `json:"createTime"` } // PipelineBase is the base info of pipeline @@ -1580,7 +1584,6 @@ type RunStat struct { // CreatePipelineRequest is the request body of creating pipeline type CreatePipelineRequest struct { Name string `json:"name" validate:"checkname"` - Project string `json:"project"` Alias string `json:"alias" validate:"checkalias" optional:"true"` Description string `json:"description" optional:"true"` Spec workflowv1alpha1.WorkflowSpec `json:"spec"` @@ -1593,8 +1596,9 @@ type PipelineMetaResponse struct { // ListPipelineRequest is the request body of listing pipeline type ListPipelineRequest struct { - Projects []string `json:"projects"` - Query string `json:"query"` + Projects []string `json:"projects" optional:"true"` + Query string `json:"query" optional:"true"` + Detailed bool `json:"detailed" optional:"true"` } // ListPipelineResponse is the response body of listing pipeline @@ -1616,11 +1620,6 @@ type UpdatePipelineRequest struct { Spec workflowv1alpha1.WorkflowSpec `json:"spec" optional:"true"` } -// GetPipelineRequest is the request body of getting pipeline -type GetPipelineRequest struct { - Detailed bool `json:"detailed"` -} - // GetPipelineResponse is the response body of getting pipeline type GetPipelineResponse struct { PipelineBase `json:",inline"` @@ -1629,9 +1628,8 @@ type GetPipelineResponse struct { // PipelineInfo is the info of pipeline type PipelineInfo struct { - RelatedApps []ApplicationBase `json:"relatedApps"` - LastRunStatus workflowv1alpha1.WorkflowRunStatus `json:"lastRunStatus"` - RunStat RunStat `json:"runStat"` + LastRun *PipelineRun `json:"lastRun"` + RunStat RunStat `json:"runStat"` } /***********************/ @@ -1652,9 +1650,9 @@ type PipelineRunBriefing struct { // PipelineRunMeta is the metadata of pipeline run type PipelineRunMeta struct { - PipelineName string `json:"pipelineName"` - Project string `json:"project"` - PipelineRunName string `json:"pipelineRunName"` + PipelineName string `json:"pipelineName"` + Project NameAlias `json:"project"` + PipelineRunName string `json:"pipelineRunName"` } // PipelineRun is the info of pipeline run @@ -1667,14 +1665,16 @@ type PipelineRun struct { type PipelineRunBase struct { PipelineRunMeta `json:",inline"` // Record marks the run of the pipeline - Record int64 `json:"record"` - ContextName string `json:"contextName"` - Spec workflowv1alpha1.WorkflowRunSpec `json:"spec"` + Record int64 `json:"record"` + ContextName string `json:"contextName"` + ContextValues []model.Value `json:"contextValues"` + Spec workflowv1alpha1.WorkflowRunSpec `json:"spec"` } // RunPipelineRequest is the request body of running pipeline type RunPipelineRequest struct { // Mode is the mode of the pipeline run. Available values are: "StepByStep", "DAG" for both `step` and `subStep` + // default: "StepByStep" for `step`, "DAG" for `subStep` Mode workflowv1alpha1.WorkflowExecuteMode `json:"mode" optional:"true"` ContextName string `json:"contextName"` } @@ -1687,12 +1687,18 @@ type ListPipelineRunResponse struct { // GetPipelineRunLogResponse is the response body of getting pipeline run log type GetPipelineRunLogResponse struct { - Log []Log `json:"log"` + StepBase `json:",inline"` + Log string `json:"log"` } // GetPipelineRunOutputResponse is the response body of getting pipeline run output type GetPipelineRunOutputResponse struct { - Output []Output `json:"output"` + StepOutputs []StepOutputBase `json:"outputs"` +} + +// GetPipelineRunInputResponse is the response body of getting pipeline run input +type GetPipelineRunInputResponse struct { + StepInputs []StepInputBase `json:"inputs"` } // StepBase is the base info of step @@ -1703,16 +1709,31 @@ type StepBase struct { Phase string `json:"phase"` } -// Log is the log of step -type Log struct { +// StepOutputBase is the output of step +type StepOutputBase struct { StepBase `json:",inline"` - Log string `json:"log"` + Values []OutputVar `json:"values"` } -// Output is the output of step -type Output struct { +// StepInputBase is the input of step +type StepInputBase struct { StepBase `json:",inline"` - Vars map[string]string `json:"vars"` + Values []InputVar `json:"values"` +} + +// OutputVar is one output var +type OutputVar struct { + Name string `json:"name"` + Value string `json:"value"` + ValueFrom string `json:"valueFrom"` +} + +// InputVar is one input var +type InputVar struct { + From string `json:"from"` + FromStep string `json:"fromStep"` + ParameterKey string `json:"parameterKey"` + Value string `json:"value"` } /*******************/ diff --git a/pkg/apiserver/interfaces/api/interfaces.go b/pkg/apiserver/interfaces/api/interfaces.go index e44cc302c..c66f07adc 100644 --- a/pkg/apiserver/interfaces/api/interfaces.go +++ b/pkg/apiserver/interfaces/api/interfaces.go @@ -62,6 +62,7 @@ func InitAPIBean() []interface{} { RegisterAPIInterface(NewApplicationAPIInterface()) RegisterAPIInterface(NewProjectAPIInterface()) RegisterAPIInterface(NewEnvAPIInterface()) + RegisterAPIInterface(NewPipelineAPIInterface()) // Extension RegisterAPIInterface(NewDefinitionAPIInterface()) @@ -82,7 +83,6 @@ func InitAPIBean() []interface{} { RegisterAPIInterface(NewWebhookAPIInterface()) RegisterAPIInterface(NewRepositoryAPIInterface()) RegisterAPIInterface(NewCloudShellAPIInterface()) - RegisterAPIInterface(NewPipelineAPIInterface()) // Authentication RegisterAPIInterface(NewAuthenticationAPIInterface()) diff --git a/pkg/apiserver/interfaces/api/pipeline.go b/pkg/apiserver/interfaces/api/pipeline.go index 95e04847b..695a19fc4 100644 --- a/pkg/apiserver/interfaces/api/pipeline.go +++ b/pkg/apiserver/interfaces/api/pipeline.go @@ -18,199 +18,242 @@ package api import ( "context" + "strconv" restfulspec "github.com/emicklei/go-restful-openapi/v2" "github.com/emicklei/go-restful/v3" - workflowv1alpha1 "github.com/kubevela/workflow/api/v1alpha1" - - "github.com/oam-dev/kubevela/pkg/apiserver/utils/log" + "github.com/kubevela/workflow/api/v1alpha1" + "github.com/pkg/errors" "github.com/oam-dev/kubevela/pkg/apiserver/domain/service" apis "github.com/oam-dev/kubevela/pkg/apiserver/interfaces/api/dto/v1" "github.com/oam-dev/kubevela/pkg/apiserver/utils/bcode" + "github.com/oam-dev/kubevela/pkg/apiserver/utils/log" ) -type pipelineAPIInterface struct { - PipelineService service.PipelineService `inject:""` - PipelineRunService service.PipelineRunService `inject:""` - ContextService service.ContextService `inject:""` -} - -type pipelinePathParamKey string - const ( // Project is the project name key of query param - Project pipelinePathParamKey = "projectName" + Project string = "projectName" // Pipeline is the pipeline name of query param - Pipeline pipelinePathParamKey = "pipelineName" + Pipeline string = "pipelineName" // PipelineRun is the pipeline run name of query param - PipelineRun pipelinePathParamKey = "runName" + PipelineRun string = "runName" // ContextName is the context name of query param - ContextName pipelinePathParamKey = "contextName" + ContextName string = "contextName" ) -// GetWebServiceRoute is the implementation of pipeline Interface -func (p *pipelineAPIInterface) GetWebServiceRoute() *restful.WebService { - - ws := new(restful.WebService) +func initPipelineRoutes(ws *restful.WebService, n *projectAPIInterface) { tags := []string{"pipeline"} - projParam := func(builder *restful.RouteBuilder) { - builder.Param(ws.QueryParameter(string(Project), "project name").Required(true)) + builder.Param(ws.PathParameter(Project, "project name").Required(true)) + builder.Filter(n.projectCheckFilter) } pipelineParam := func(builder *restful.RouteBuilder) { - builder.Param(ws.PathParameter(string(Pipeline), "pipeline name").Required(true)) - builder.Filter(p.pipelineCheckFilter) + builder.Param(ws.PathParameter(Pipeline, "pipeline name").Required(true)) + builder.Filter(n.pipelineCheckFilter) } ctxParam := func(builder *restful.RouteBuilder) { - builder.Param(ws.PathParameter(string(ContextName), "pipeline context name").Required(true)) - builder.Filter(p.pipelineContextCheckFilter) + builder.Param(ws.PathParameter(ContextName, "pipeline context name").Required(true)) + builder.Filter(n.pipelineContextCheckFilter) } runParam := func(builder *restful.RouteBuilder) { - builder.Param(ws.PathParameter(string(PipelineRun), "pipeline run name").Required(true)) - builder.Filter(p.pipelineRunCheckFilter) + builder.Param(ws.PathParameter(PipelineRun, "pipeline run name").Required(true)) + builder.Filter(n.pipelineRunCheckFilter) } meta := func(builder *restful.RouteBuilder) { builder.Metadata(restfulspec.KeyOpenAPITags, tags) } - ws.Path(versionPrefix+"/pipelines"). - Consumes(restful.MIME_JSON, restful.MIME_XML). - Produces(restful.MIME_JSON, restful.MIME_XML). - Doc("api for pipeline manage") - - ws.Route(ws.POST("").To(p.createPipeline). + ws.Route(ws.POST("/{projectName}/pipelines").To(n.createPipeline). Doc("create pipeline"). Reads(apis.CreatePipelineRequest{}). Returns(200, "OK", apis.PipelineBase{}). Returns(400, "Bad Request", bcode.Bcode{}). - Writes(apis.PipelineBase{}).Do(meta)) + Filter(n.RBACService.CheckPerm("project/pipeline", "create")). + Writes(apis.PipelineBase{}).Do(meta, projParam)) - ws.Route(ws.GET("").To(p.listPipelines). - Doc("list pipelines"). - Param(ws.QueryParameter("query", "Fuzzy search based on name or description").DataType("string")). - Returns(200, "OK", apis.ListPipelineResponse{}). - Returns(400, "Bad Request", bcode.Bcode{}). - Writes(apis.ListPipelineResponse{}).Do(meta, projParam)) - - ws.Route(ws.GET("/{pipelineName}").To(p.getPipeline). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}").To(n.getPipeline). Doc("get pipeline"). - Reads(apis.GetPipelineRequest{}). Returns(200, "OK", apis.GetPipelineResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). - Writes(apis.GetPipelineResponse{}).Do(meta, projParam, pipelineParam)) + // use Param instead of pipelineParam to get pipeline information + Param(ws.PathParameter(Pipeline, "pipeline name").Required(true)). + Filter(n.RBACService.CheckPerm("project/pipeline", "detail")). + Writes(apis.GetPipelineResponse{}).Do(meta, projParam)) - ws.Route(ws.PUT("/{pipelineName}").To(p.updatePipeline). + ws.Route(ws.PUT("/{projectName}/pipelines/{pipelineName}").To(n.updatePipeline). Doc("update pipeline"). Reads(apis.UpdatePipelineRequest{}). Returns(200, "OK", apis.PipelineBase{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline", "update")). Writes(apis.PipelineBase{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.DELETE("/{pipelineName}").To(p.deletePipeline). + ws.Route(ws.DELETE("/{projectName}/pipelines/{pipelineName}").To(n.deletePipeline). Doc("delete pipeline"). Returns(200, "OK", apis.PipelineMetaResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline", "delete")). Writes(apis.PipelineMetaResponse{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.POST("/{pipelineName}/contexts").To(p.createContextValue). + ws.Route(ws.POST("/{projectName}/pipelines/{pipelineName}/contexts").To(n.createContextValue). Doc("create pipeline context values"). Reads(apis.CreateContextValuesRequest{}). Returns(200, "OK", apis.Context{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/context", "create")). Writes(apis.Context{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.GET("/{pipelineName}/contexts").To(p.listContextValues). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/contexts").To(n.listContextValues). Doc("list pipeline context values"). Returns(200, "OK", apis.ListContextValueResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/context", "list")). Writes(apis.ListContextValueResponse{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.PUT("/{pipelineName}/contexts/{contextName}").To(p.updateContextValue). + ws.Route(ws.PUT("/{projectName}/pipelines/{pipelineName}/contexts/{contextName}").To(n.updateContextValue). Doc("update pipeline context value"). Reads(apis.UpdateContextValuesRequest{}). Returns(200, "OK", apis.Context{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/context", "update")). Writes(apis.Context{}).Do(meta, projParam, pipelineParam, ctxParam)) - ws.Route(ws.DELETE("/{pipelineName}/contexts/{contextName}").To(p.deleteContextValue). + ws.Route(ws.DELETE("/{projectName}/pipelines/{pipelineName}/contexts/{contextName}").To(n.deleteContextValue). Doc("delete pipeline context value"). Returns(200, "OK", apis.ContextNameResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/context", "delete")). Writes(apis.ContextNameResponse{}).Do(meta, projParam, pipelineParam, ctxParam)) - ws.Route(ws.POST("/{pipelineName}/run").To(p.runPipeline). + ws.Route(ws.POST("/{projectName}/pipelines/{pipelineName}/run").To(n.runPipeline). Doc("run pipeline"). Reads(apis.RunPipelineRequest{}). - Returns(200, "OK", apis.PipelineRunMeta{}). + Returns(200, "OK", apis.PipelineRun{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline", "run")). Writes(apis.PipelineRunMeta{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.GET("/{pipelineName}/runs").To(p.listPipelineRuns). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs").To(n.listPipelineRuns). Doc("list pipeline runs"). Param(ws.QueryParameter("status", "query identifier of the status").DataType("string")). Returns(200, "OK", apis.ListPipelineRunResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "list")). Writes(apis.ListPipelineRunResponse{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.POST("/{pipelineName}/runs/{runName}/stop").To(p.stopPipeline). + ws.Route(ws.POST("/{projectName}/pipelines/{pipelineName}/runs/{runName}/stop").To(n.stopPipeline). Doc("stop pipeline run"). Returns(200, "OK", apis.PipelineRunMeta{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "stop")). Writes(apis.PipelineRunMeta{}).Do(meta, projParam, pipelineParam, runParam)) - ws.Route(ws.GET("/{pipelineName}/runs/{runName}").To(p.getPipelineRun). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs/{runName}").To(n.getPipelineRun). Doc("get pipeline run"). Returns(200, "OK", apis.PipelineRunBase{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "get")). Writes(apis.PipelineRunBase{}).Do(meta, projParam, pipelineParam, runParam)) - ws.Route(ws.DELETE("/{pipelineName}/runs/{runName}").To(p.deletePipelineRun). + ws.Route(ws.DELETE("/{projectName}/pipelines/{pipelineName}/runs/{runName}").To(n.deletePipelineRun). Doc("delete pipeline run"). Returns(200, "OK", apis.PipelineRunMeta{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "delete")). Writes(apis.PipelineRunMeta{}).Do(meta, projParam, pipelineParam, runParam)) // get pipeline run status - ws.Route(ws.GET("/{pipelineName}/runs/{runName}/status").To(p.getPipelineRunStatus). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs/{runName}/status").To(n.getPipelineRunStatus). Doc("get pipeline run status"). - Returns(200, "OK", workflowv1alpha1.WorkflowRunStatus{}). + Returns(200, "OK", v1alpha1.WorkflowRunStatus{}). Returns(400, "Bad Request", bcode.Bcode{}). - Writes(workflowv1alpha1.WorkflowRunStatus{}).Do(meta, projParam, pipelineParam, runParam)) + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "detail")). + Writes(v1alpha1.WorkflowRunStatus{}).Do(meta, projParam, pipelineParam, runParam)) // get pipeline run log - ws.Route(ws.GET("/{pipelineName}/runs/{runName}/log").To(p.getPipelineRunLog). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs/{runName}/log").To(n.getPipelineRunLog). Doc("get pipeline run log"). - Param(ws.QueryParameter("step", "query by specific id").DataType("string")). + Param(ws.QueryParameter("step", "query by specific step name").DataType("string")). Returns(200, "OK", apis.GetPipelineRunLogResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "detail")). Writes(apis.GetPipelineRunLogResponse{}).Do(meta, projParam, pipelineParam, runParam)) // get pipeline run output - ws.Route(ws.GET("/{pipelineName}/runs/{runName}/output").To(p.getPipelineRunOutput). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs/{runName}/output").To(n.getPipelineRunOutput). Doc("get pipeline run output"). - Param(ws.QueryParameter("step", "query by specific id").DataType("string")). + Param(ws.QueryParameter("step", "query by specific step name").DataType("string").Required(true)). Returns(200, "OK", apis.GetPipelineRunOutputResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "detail")). Writes(apis.GetPipelineRunOutputResponse{}).Do(meta, projParam, pipelineParam, runParam)) + // get pipeline run input + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs/{runName}/input").To(n.getPipelineRunInput). + Doc("get pipeline run input"). + Param(ws.QueryParameter("step", "query by specific step name").DataType("string").Required(true)). + Returns(200, "OK", apis.GetPipelineRunInputResponse{}). + Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "detail")). + Writes(apis.GetPipelineRunInputResponse{}).Do(meta, projParam, pipelineParam, runParam)) + + ws.Filter(authCheckFilter) +} + +// GetWebServiceRoute is the implementation of pipeline Interface +func (n *pipelineAPIInterface) GetWebServiceRoute() *restful.WebService { + tags := []string{"pipeline"} + meta := func(builder *restful.RouteBuilder) { + builder.Metadata(restfulspec.KeyOpenAPITags, tags) + } + + ws := new(restful.WebService) + + ws.Path(versionPrefix+"/pipelines"). + Consumes(restful.MIME_XML, restful.MIME_JSON). + Produces(restful.MIME_JSON, restful.MIME_XML). + Doc("api for project manage") + + ws.Route(ws.GET("").To(n.listPipelines). + Doc("list pipelines"). + Param(ws.QueryParameter("query", "Fuzzy search based on name or description").DataType("string")). + Param(ws.QueryParameter("projectName", "query pipelines within a project").DataType("string")). + Param(ws.QueryParameter("detailed", "query pipelines with detail").DataType("bool").DefaultValue("true")). + Returns(200, "OK", apis.ListPipelineResponse{}). + Returns(400, "Bad Request", bcode.Bcode{}). + Writes(apis.ListPipelineResponse{}).Do(meta)) + ws.Filter(authCheckFilter) return ws } +type pipelineAPIInterface struct { + PipelineService service.PipelineService `inject:""` +} + // NewPipelineAPIInterface new pipeline manage APIInterface func NewPipelineAPIInterface() Interface { return &pipelineAPIInterface{} } -func (p *pipelineAPIInterface) listPipelines(req *restful.Request, res *restful.Response) { - var projetNames []string - if req.QueryParameter("project") != "" { - projetNames = append(projetNames, req.QueryParameter("project")) +func (n *pipelineAPIInterface) listPipelines(req *restful.Request, res *restful.Response) { + var projectNames []string + if req.QueryParameter(Project) != "" { + projectNames = append(projectNames, req.QueryParameter(Project)) } - pipelines, err := p.PipelineService.ListPipelines(req.Request.Context(), apis.ListPipelineRequest{ - Projects: projetNames, + _detailed := req.QueryParameter("detailed") + if _detailed == "" { + _detailed = "true" + } + detailed, err := strconv.ParseBool(_detailed) + if err != nil { + bcode.ReturnError(req, res, errors.Wrap(err, "invalid detailed param")) + } + pipelines, err := n.PipelineService.ListPipelines(req.Request.Context(), apis.ListPipelineRequest{ + Projects: projectNames, Query: req.QueryParameter("query"), + Detailed: detailed, }) if err != nil { log.Logger.Errorf("list pipeline failure %s", err.Error()) @@ -223,15 +266,18 @@ func (p *pipelineAPIInterface) listPipelines(req *restful.Request, res *restful. } } -func (p *pipelineAPIInterface) getPipeline(req *restful.Request, res *restful.Response) { - pipeline := req.Request.Context().Value(apis.CtxKeyPipeline).(apis.PipelineBase) +func (n *projectAPIInterface) getPipeline(req *restful.Request, res *restful.Response) { + pipeline, err := n.PipelineService.GetPipeline(req.Request.Context(), req.PathParameter(Pipeline), true) + if err != nil { + return + } if err := res.WriteEntity(pipeline); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) createPipeline(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) createPipeline(req *restful.Request, res *restful.Response) { var createReq apis.CreatePipelineRequest if err := req.ReadEntity(&createReq); err != nil { bcode.ReturnError(req, res, err) @@ -241,19 +287,25 @@ func (p *pipelineAPIInterface) createPipeline(req *restful.Request, res *restful bcode.ReturnError(req, res, err) return } - pipelineBase, err := p.PipelineService.CreatePipeline(req.Request.Context(), createReq) + pipelineBase, err := n.PipelineService.CreatePipeline(req.Request.Context(), createReq) if err != nil { log.Logger.Errorf("create pipeline failure %s", err.Error()) bcode.ReturnError(req, res, err) return } + _, err = n.ContextService.InitContext(req.Request.Context(), pipelineBase.Project.Name, pipelineBase.Name) + if err != nil { + log.Logger.Errorf("init pipeline context failure: %s", err.Error()) + bcode.ReturnError(req, res, err) + return + } if err := res.WriteEntity(pipelineBase); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) updatePipeline(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) updatePipeline(req *restful.Request, res *restful.Response) { var updateReq apis.UpdatePipelineRequest if err := req.ReadEntity(&updateReq); err != nil { bcode.ReturnError(req, res, err) @@ -263,8 +315,8 @@ func (p *pipelineAPIInterface) updatePipeline(req *restful.Request, res *restful bcode.ReturnError(req, res, err) return } - base := req.Request.Context().Value(apis.CtxKeyPipeline).(apis.PipelineBase) - pipelineBase, err := p.PipelineService.UpdatePipeline(req.Request.Context(), base.Name, base.Project, updateReq) + pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) + pipelineBase, err := n.PipelineService.UpdatePipeline(req.Request.Context(), pipeline.Name, updateReq) if err != nil { log.Logger.Errorf("update pipeline failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -276,43 +328,43 @@ func (p *pipelineAPIInterface) updatePipeline(req *restful.Request, res *restful } } -func (p *pipelineAPIInterface) deletePipeline(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) deletePipeline(req *restful.Request, res *restful.Response) { pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) - err := p.PipelineService.DeletePipeline(req.Request.Context(), pipeline) + err := n.PipelineService.DeletePipeline(req.Request.Context(), pipeline) if err != nil { log.Logger.Errorf("delete pipeline failure %s", err.Error()) bcode.ReturnError(req, res, err) return } - if err := res.WriteEntity(pipeline.PipelineMeta); err != nil { + if err := res.WriteEntity(apis.EmptyResponse{}); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) runPipeline(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) runPipeline(req *restful.Request, res *restful.Response) { var runReq apis.RunPipelineRequest pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) - if err := req.ReadEntity(runReq); err != nil { + if err := req.ReadEntity(&runReq); err != nil { bcode.ReturnError(req, res, err) return } - err := p.PipelineService.RunPipeline(req.Request.Context(), pipeline, runReq) + run, err := n.PipelineService.RunPipeline(req.Request.Context(), pipeline, runReq) if err != nil { log.Logger.Errorf("run pipeline failure %s", err.Error()) bcode.ReturnError(req, res, err) return } - if err := res.WriteEntity(pipeline.PipelineMeta); err != nil { + if err := res.WriteEntity(run); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) stopPipeline(req *restful.Request, res *restful.Response) { - pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(apis.PipelineRun) - err := p.PipelineRunService.StopPipelineRun(req.Request.Context(), pipelineRun.PipelineRunBase) +func (n *projectAPIInterface) stopPipeline(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) + err := n.PipelineRunService.StopPipelineRun(req.Request.Context(), pipelineRun.PipelineRunBase) if err != nil { log.Logger.Errorf("stop pipeline failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -324,9 +376,9 @@ func (p *pipelineAPIInterface) stopPipeline(req *restful.Request, res *restful.R } } -func (p *pipelineAPIInterface) listPipelineRuns(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) listPipelineRuns(req *restful.Request, res *restful.Response) { pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) - pipelineRuns, err := p.PipelineRunService.ListPipelineRuns(req.Request.Context(), pipeline) + pipelineRuns, err := n.PipelineRunService.ListPipelineRuns(req.Request.Context(), pipeline) if err != nil { log.Logger.Errorf("list pipeline runs failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -338,33 +390,65 @@ func (p *pipelineAPIInterface) listPipelineRuns(req *restful.Request, res *restf } } -func (p *pipelineAPIInterface) getPipelineRun(req *restful.Request, res *restful.Response) { - pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(apis.PipelineRun) +func (n *projectAPIInterface) getPipelineRun(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) if err := res.WriteEntity(pipelineRun.PipelineRunBase); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) getPipelineRunStatus(req *restful.Request, res *restful.Response) { - pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(apis.PipelineRun) +func (n *projectAPIInterface) getPipelineRunStatus(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) if err := res.WriteEntity(pipelineRun.Status); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) getPipelineRunLog(req *restful.Request, res *restful.Response) { - +func (n *projectAPIInterface) getPipelineRunLog(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) + logs, err := n.PipelineRunService.GetPipelineRunLog(req.Request.Context(), *pipelineRun, req.QueryParameter("step")) + if err != nil { + log.Logger.Errorf("get pipeline run log failure %s", err.Error()) + bcode.ReturnError(req, res, err) + return + } + if err := res.WriteEntity(logs); err != nil { + bcode.ReturnError(req, res, err) + return + } } -func (p *pipelineAPIInterface) getPipelineRunOutput(req *restful.Request, res *restful.Response) { - +func (n *projectAPIInterface) getPipelineRunOutput(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) + output, err := n.PipelineRunService.GetPipelineRunOutput(req.Request.Context(), *pipelineRun, req.QueryParameter("step")) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + if err := res.WriteEntity(output); err != nil { + bcode.ReturnError(req, res, err) + return + } } -func (p *pipelineAPIInterface) deletePipelineRun(req *restful.Request, res *restful.Response) { - pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(apis.PipelineRun) - err := p.PipelineRunService.DeletePipelineRun(req.Request.Context(), pipelineRun.PipelineRunMeta) +func (n *projectAPIInterface) getPipelineRunInput(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) + input, err := n.PipelineRunService.GetPipelineRunInput(req.Request.Context(), *pipelineRun, req.QueryParameter("step")) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + if err := res.WriteEntity(input); err != nil { + bcode.ReturnError(req, res, err) + return + } +} + +func (n *projectAPIInterface) deletePipelineRun(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) + err := n.PipelineRunService.DeletePipelineRun(req.Request.Context(), pipelineRun.PipelineRunMeta) if err != nil { log.Logger.Errorf("delete pipeline run failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -376,11 +460,11 @@ func (p *pipelineAPIInterface) deletePipelineRun(req *restful.Request, res *rest } } -func (p *pipelineAPIInterface) listContextValues(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) listContextValues(req *restful.Request, res *restful.Response) { pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) - contextValues, err := p.ContextService.ListContexts(req.Request.Context(), pipeline.Project, pipeline.Name) + contextValues, err := n.ContextService.ListContexts(req.Request.Context(), pipeline.Project.Name, pipeline.Name) if err != nil { - log.Logger.Errorf("list context values failure %s", err.Error()) + log.Logger.Errorf("list context values failure: %s", err.Error()) bcode.ReturnError(req, res, err) return } @@ -390,7 +474,7 @@ func (p *pipelineAPIInterface) listContextValues(req *restful.Request, res *rest } } -func (p *pipelineAPIInterface) createContextValue(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) createContextValue(req *restful.Request, res *restful.Response) { pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) var createReq apis.CreateContextValuesRequest if err := req.ReadEntity(&createReq); err != nil { @@ -403,7 +487,7 @@ func (p *pipelineAPIInterface) createContextValue(req *restful.Request, res *res } pipelineCtx := apis.Context(createReq) - _, err := p.ContextService.CreateContext(req.Request.Context(), pipeline.Project, pipeline.Name, pipelineCtx) + _, err := n.ContextService.CreateContext(req.Request.Context(), pipeline.Project.Name, pipeline.Name, pipelineCtx) if err != nil { log.Logger.Errorf("create context failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -415,8 +499,8 @@ func (p *pipelineAPIInterface) createContextValue(req *restful.Request, res *res } } -func (p *pipelineAPIInterface) updateContextValue(req *restful.Request, res *restful.Response) { - plCtx := req.Request.Context().Value(&apis.CtxKeyPipelineContex).(apis.Context) +func (n *projectAPIInterface) updateContextValue(req *restful.Request, res *restful.Response) { + plCtx := req.Request.Context().Value(&apis.CtxKeyPipelineContext).(apis.Context) pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) var updateReq apis.UpdateContextValuesRequest if err := req.ReadEntity(&updateReq); err != nil { @@ -428,7 +512,7 @@ func (p *pipelineAPIInterface) updateContextValue(req *restful.Request, res *res return } pipelineCtx := apis.Context{Name: plCtx.Name, Values: updateReq.Values} - _, err := p.ContextService.UpdateContext(req.Request.Context(), pipeline.Project, pipeline.Name, pipelineCtx) + _, err := n.ContextService.UpdateContext(req.Request.Context(), pipeline.Project.Name, pipeline.Name, pipelineCtx) if err != nil { log.Logger.Errorf("update context failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -440,10 +524,10 @@ func (p *pipelineAPIInterface) updateContextValue(req *restful.Request, res *res } } -func (p *pipelineAPIInterface) deleteContextValue(req *restful.Request, res *restful.Response) { - plCtx := req.Request.Context().Value(&apis.CtxKeyPipelineContex).(apis.Context) +func (n *projectAPIInterface) deleteContextValue(req *restful.Request, res *restful.Response) { + plCtx := req.Request.Context().Value(&apis.CtxKeyPipelineContext).(apis.Context) pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) - err := p.ContextService.DeleteContext(req.Request.Context(), pipeline.Project, pipeline.Name, plCtx.Name) + err := n.ContextService.DeleteContext(req.Request.Context(), pipeline.Project.Name, pipeline.Name, plCtx.Name) if err != nil { log.Logger.Errorf("delete context failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -455,8 +539,18 @@ func (p *pipelineAPIInterface) deleteContextValue(req *restful.Request, res *res } } -func (p *pipelineAPIInterface) pipelineCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { - pipeline, err := p.PipelineService.GetPipeline(req.Request.Context(), req.PathParameter("pipelineName"), req.QueryParameter("projectName")) +func (n *projectAPIInterface) projectCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { + project, err := n.ProjectService.GetProject(req.Request.Context(), req.PathParameter(Project)) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + req.Request = req.Request.WithContext(context.WithValue(req.Request.Context(), &apis.CtxKeyProject, project)) + chain.ProcessFilter(req, res) +} + +func (n *projectAPIInterface) pipelineCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { + pipeline, err := n.PipelineService.GetPipeline(req.Request.Context(), req.PathParameter(Pipeline), false) if err != nil { bcode.ReturnError(req, res, err) return @@ -465,8 +559,8 @@ func (p *pipelineAPIInterface) pipelineCheckFilter(req *restful.Request, res *re chain.ProcessFilter(req, res) } -func (p *pipelineAPIInterface) pipelineContextCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { - contexts, err := p.ContextService.ListContexts(req.Request.Context(), req.PathParameter("pipelineName"), req.QueryParameter("projectName")) +func (n *projectAPIInterface) pipelineContextCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { + contexts, err := n.ContextService.ListContexts(req.Request.Context(), req.PathParameter(Project), req.PathParameter(Pipeline)) if err != nil { bcode.ReturnError(req, res, err) return @@ -477,20 +571,22 @@ func (p *pipelineAPIInterface) pipelineContextCheckFilter(req *restful.Request, bcode.ReturnError(req, res, bcode.ErrContextNotFound) return } - req.Request = req.Request.WithContext(context.WithValue(req.Request.Context(), &apis.CtxKeyPipelineContex, apis.Context{ + req.Request = req.Request.WithContext(context.WithValue(req.Request.Context(), &apis.CtxKeyPipelineContext, apis.Context{ Name: contextName, Values: contextValue, })) chain.ProcessFilter(req, res) } -func (p *pipelineAPIInterface) pipelineRunCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { +func (n *projectAPIInterface) pipelineRunCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { meta := apis.PipelineRunMeta{ - PipelineName: req.PathParameter(string(Pipeline)), - Project: req.QueryParameter(string(Project)), - PipelineRunName: req.PathParameter(string(PipelineRun)), + PipelineName: req.PathParameter(Pipeline), + Project: apis.NameAlias{ + Name: req.PathParameter(Project), + }, + PipelineRunName: req.PathParameter(PipelineRun), } - run, err := p.PipelineRunService.GetPipelineRun(req.Request.Context(), meta) + run, err := n.PipelineRunService.GetPipelineRun(req.Request.Context(), meta) if err != nil { bcode.ReturnError(req, res, err) return diff --git a/pkg/apiserver/interfaces/api/project.go b/pkg/apiserver/interfaces/api/project.go index 8a7c548de..032d5bda4 100644 --- a/pkg/apiserver/interfaces/api/project.go +++ b/pkg/apiserver/interfaces/api/project.go @@ -29,10 +29,14 @@ import ( ) type projectAPIInterface struct { - RbacService service.RBACService `inject:""` - ProjectService service.ProjectService `inject:""` - TargetService service.TargetService `inject:""` - ConfigService service.ConfigService `inject:""` + RbacService service.RBACService `inject:""` + ProjectService service.ProjectService `inject:""` + TargetService service.TargetService `inject:""` + ConfigService service.ConfigService `inject:""` + PipelineService service.PipelineService `inject:""` + PipelineRunService service.PipelineRunService `inject:""` + ContextService service.ContextService `inject:""` + RBACService service.RBACService `inject:""` } // NewProjectAPIInterface new project APIInterface @@ -305,6 +309,7 @@ func (n *projectAPIInterface) GetWebServiceRoute() *restful.WebService { Returns(400, "Bad Request", bcode.Bcode{}). Writes(apis.ListTerraformProviderResponse{})) + initPipelineRoutes(ws, n) ws.Filter(authCheckFilter) return ws } diff --git a/pkg/apiserver/utils/bcode/017_pipeline.go b/pkg/apiserver/utils/bcode/017_pipeline.go index 6b31f5ad6..635b0cf95 100644 --- a/pkg/apiserver/utils/bcode/017_pipeline.go +++ b/pkg/apiserver/utils/bcode/017_pipeline.go @@ -21,4 +21,22 @@ var ( ErrContextNotFound = NewBcode(400, 17001, "pipeline context is not found") // ErrContextAlreadyExist means the certain context already exists ErrContextAlreadyExist = NewBcode(400, 17002, "pipeline context of pipeline already exist") + // ErrGetPipelineInfo means failed to get pipeline info + ErrGetPipelineInfo = NewBcode(400, 17003, "get pipeline info failed") + // ErrPipelineNotExist means specific pipeline not found + ErrPipelineNotExist = NewBcode(404, 17004, "failed to find log pods") + // ErrGetPodsLogs means failed to get pods logs + ErrGetPodsLogs = NewBcode(500, 17006, "failed to get pods logs") + // ErrReadSourceLog means failed to read source log + ErrReadSourceLog = NewBcode(500, 17007, "failed to read log from URL source") + // ErrGetContextBackendData means failed to get context backend data + ErrGetContextBackendData = NewBcode(500, 17008, "failed to get context backend data") + // ErrNoSteps means pipeline doesn't have a step + ErrNoSteps = NewBcode(400, 17009, "pipeline step number is zero") + // ErrPipelineExist means the pipeline is exist + ErrPipelineExist = NewBcode(400, 17010, "the pipeline is exist") + // ErrPipelineRunFinished means pipeline run is finished + ErrPipelineRunFinished = NewBcode(400, 17011, "pipeline run is finished") + // ErrWrongMode means the pipeline run mode is wrong + ErrWrongMode = NewBcode(400, 17012, "wrong pipeline run mode, only \"DAG\" and \"StepByStep\" are supported") ) diff --git a/pkg/utils/k8s.go b/pkg/utils/k8s.go index 4eba13c82..948ffab68 100644 --- a/pkg/utils/k8s.go +++ b/pkg/utils/k8s.go @@ -21,7 +21,19 @@ import ( "encoding/json" "fmt" "os" + "regexp" "strings" + "text/template" + "time" + + "github.com/fatih/color" + "github.com/pkg/errors" + "github.com/wercker/stern/stern" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/client-go/kubernetes" + + querytypes "github.com/oam-dev/kubevela/pkg/velaql/providers/query/types" authv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" @@ -199,3 +211,125 @@ func IsClusterScope(gvk schema.GroupVersionKind, mapper meta.RESTMapper) (bool, isClusterScope := len(mappings) > 0 && mappings[0].Scope.Name() == meta.RESTScopeNameRoot return isClusterScope, err } + +// GetPodsLogs get logs from pods +func GetPodsLogs(ctx context.Context, config *rest.Config, containerName string, selectPods []*querytypes.PodBase, tmpl string, logC chan<- string, tailLines *int64) error { + if err := verifyPods(selectPods); err != nil { + return err + } + podRegex := getPodRegex(selectPods) + pods, err := regexp.Compile(podRegex) + if err != nil { + return fmt.Errorf("fail to compile '%s' for logs query", podRegex) + } + container := regexp.MustCompile(".*") + if containerName != "" { + container = regexp.MustCompile(containerName + ".*") + } + // These pods are from the same namespace, so we can use the first one to get the namespace + namespace := selectPods[0].Metadata.Namespace + selector := labels.NewSelector() + // Only use the labels to select pod if query one pod's log. It is only used when query vela-core log + if len(selectPods) == 1 { + for k, v := range selectPods[0].Metadata.Labels { + req, _ := labels.NewRequirement(k, selection.Equals, []string{v}) + if req != nil { + selector = selector.Add(*req) + } + } + } + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + return err + } + added, removed, err := stern.Watch(ctx, + clientSet.CoreV1().Pods(namespace), + pods, + container, + nil, + []stern.ContainerState{stern.RUNNING, stern.TERMINATED}, + selector, + ) + if err != nil { + return err + } + tails := make(map[string]*stern.Tail) + + funs := map[string]interface{}{ + "json": func(in interface{}) (string, error) { + b, err := json.Marshal(in) + if err != nil { + return "", err + } + return string(b), nil + }, + "color": func(color color.Color, text string) string { + return color.SprintFunc()(text) + }, + } + template, err := template.New("log").Funcs(funs).Parse(tmpl) + if err != nil { + return errors.Wrap(err, "unable to parse template") + } + + go func() { + for p := range added { + id := p.GetID() + if tails[id] != nil { + continue + } + // 48h + dur, _ := time.ParseDuration("48h") + tail := stern.NewTail(p.Namespace, p.Pod, p.Container, template, &stern.TailOptions{ + Timestamps: true, + SinceSeconds: int64(dur.Seconds()), + Exclude: nil, + Include: nil, + Namespace: false, + TailLines: tailLines, // default for all logs + }) + tails[id] = tail + + tail.Start(ctx, clientSet.CoreV1().Pods(p.Namespace), logC) + } + }() + + go func() { + for p := range removed { + id := p.GetID() + if tails[id] == nil { + continue + } + tails[id].Close() + delete(tails, id) + } + }() + + <-ctx.Done() + close(logC) + return nil +} + +func getPodRegex(pods []*querytypes.PodBase) string { + var podNames []string + for _, pod := range pods { + podNames = append(podNames, fmt.Sprintf("(%s.*)", pod.Metadata.Name)) + } + return strings.Join(podNames, "|") +} + +func verifyPods(pods []*querytypes.PodBase) error { + if len(pods) == 0 { + return errors.New("no pods selected") + } + if len(pods) == 1 { + return nil + } + namespace := pods[0].Metadata.Namespace + for _, pod := range pods { + if pod.Metadata.Namespace != namespace { + return errors.New("cannot select pods from different namespaces") + } + } + return nil +} diff --git a/references/cli/logs.go b/references/cli/logs.go index d1b012eea..1fd5d23ba 100644 --- a/references/cli/logs.go +++ b/references/cli/logs.go @@ -18,24 +18,16 @@ package cli import ( "context" - "encoding/json" "fmt" - "regexp" "strings" - "text/template" - "time" "github.com/fatih/color" - "github.com/pkg/errors" "github.com/spf13/cobra" - "github.com/wercker/stern/stern" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/selection" - "k8s.io/client-go/kubernetes" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/multicluster" + "github.com/oam-dev/kubevela/pkg/utils" "github.com/oam-dev/kubevela/pkg/utils/common" "github.com/oam-dev/kubevela/pkg/utils/util" querytypes "github.com/oam-dev/kubevela/pkg/velaql/providers/query/types" @@ -99,45 +91,25 @@ type Args struct { } func (l *Args) printPodLogs(ctx context.Context, ioStreams util.IOStreams, selectPod *querytypes.PodBase, filters []string) error { - pod, err := regexp.Compile(selectPod.Metadata.Name + ".*") - if err != nil { - return fmt.Errorf("fail to compile '%s' for logs query", selectPod.Metadata.Name+".*") - } - container := regexp.MustCompile(".*") - if l.ContainerName != "" { - container = regexp.MustCompile(l.ContainerName + ".*") - } - namespace := selectPod.Metadata.Namespace - selector := labels.NewSelector() - for k, v := range selectPod.Metadata.Labels { - req, _ := labels.NewRequirement(k, selection.Equals, []string{v}) - if req != nil { - selector = selector.Add(*req) - } - } - config, err := l.Args.GetConfig() if err != nil { return err } - clientSet, err := kubernetes.NewForConfig(config) - if err != nil { - return err - } - added, removed, err := stern.Watch(ctx, - clientSet.CoreV1().Pods(namespace), - pod, - container, - nil, - []stern.ContainerState{stern.RUNNING, stern.TERMINATED}, - selector, - ) - if err != nil { - return err - } - tails := make(map[string]*stern.Tail) logC := make(chan string, 1024) + var t string + switch l.Output { + case "default": + if color.NoColor { + t = "{{.ContainerName}} {{.Message}}" + } else { + t = "{{color .ContainerColor .ContainerName}} {{.Message}}" + } + case "raw": + t = "{{.Message}}" + case "json": + t = "{{json .}}\n" + } go func() { for { select { @@ -158,71 +130,11 @@ func (l *Args) printPodLogs(ctx context.Context, ioStreams util.IOStreams, selec } }() - var t string - switch l.Output { - case "default": - if color.NoColor { - t = "{{.ContainerName}} {{.Message}}" - } else { - t = "{{color .ContainerColor .ContainerName}} {{.Message}}" - } - case "raw": - t = "{{.Message}}" - case "json": - t = "{{json .}}\n" - } - funs := map[string]interface{}{ - "json": func(in interface{}) (string, error) { - b, err := json.Marshal(in) - if err != nil { - return "", err - } - return string(b), nil - }, - "color": func(color color.Color, text string) string { - return color.SprintFunc()(text) - }, - } - template, err := template.New("log").Funcs(funs).Parse(t) + err = utils.GetPodsLogs(ctx, config, l.ContainerName, []*querytypes.PodBase{selectPod}, t, logC, nil) if err != nil { - return errors.Wrap(err, "unable to parse template") + return err } - go func() { - for p := range added { - id := p.GetID() - if tails[id] != nil { - continue - } - // 48h - dur, _ := time.ParseDuration("48h") - tail := stern.NewTail(p.Namespace, p.Pod, p.Container, template, &stern.TailOptions{ - Timestamps: true, - SinceSeconds: int64(dur.Seconds()), - Exclude: nil, - Include: nil, - Namespace: false, - TailLines: nil, // default for all logs - }) - tails[id] = tail - - tail.Start(ctx, clientSet.CoreV1().Pods(p.Namespace), logC) - } - }() - - go func() { - for p := range removed { - id := p.GetID() - if tails[id] == nil { - continue - } - tails[id].Close() - delete(tails, id) - } - }() - - <-ctx.Done() - return nil } diff --git a/test/e2e-apiserver-test/pipeline_test.go b/test/e2e-apiserver-test/pipeline_test.go new file mode 100644 index 000000000..de5b68bc7 --- /dev/null +++ b/test/e2e-apiserver-test/pipeline_test.go @@ -0,0 +1,363 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_apiserver_test + +import ( + "context" + "net/http" + "strconv" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/kubevela/workflow/api/v1alpha1" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + "github.com/oam-dev/kubevela/pkg/apiserver/domain/model" + apisv1 "github.com/oam-dev/kubevela/pkg/apiserver/interfaces/api/dto/v1" + "github.com/oam-dev/kubevela/pkg/oam/util" + "github.com/oam-dev/kubevela/pkg/utils/common" +) + +var testPipelineSteps []v1alpha1.WorkflowStep + +func init() { + rawProps := []byte(`{"url":"https://api.github.com/repos/kubevela/kubevela"}`) + testPipelineSteps = []v1alpha1.WorkflowStep{ + { + WorkflowStepBase: v1alpha1.WorkflowStepBase{ + Name: "request", + Type: "request", + Outputs: v1alpha1.StepOutputs{ + { + ValueFrom: "import \"strconv\"\n\"Current star count: \" + strconv.FormatInt(response[\"stargazers_count\"], 10)\n", + Name: "stars", + }, + }, + Properties: &runtime.RawExtension{ + Raw: rawProps, + }, + }, + }, + } +} + +var _ = Describe("Test the rest api about the pipeline", func() { + var ( + projectName1 = testNSprefix + strconv.FormatInt(time.Now().UnixNano(), 10) + pipelineName = "test-pipeline" + description = "amazing pipeline" + contextName = "test-context" + contextKey = "test-key" + contextVal = "test-val" + pipelineRunName string + ) + defer GinkgoRecover() + It("create project and apply definitions", func() { + defer GinkgoRecover() + var req = apisv1.CreateProjectRequest{ + Name: projectName1, + Description: "KubeVela Project", + } + res := post("/projects", req) + var projectBase apisv1.ProjectBase + Expect(decodeResponseBody(res, &projectBase)).Should(Succeed()) + Expect(cmp.Diff(projectBase.Name, req.Name)).Should(BeEmpty()) + Expect(cmp.Diff(projectBase.Description, req.Description)).Should(BeEmpty()) + + def1 := new(v1beta1.WorkflowStepDefinition) + def2 := new(v1beta1.WorkflowStepDefinition) + Expect(common.ReadYamlToObject("./testdata/request.yaml", def1)).Should(BeNil()) + Expect(k8sClient.Create(context.Background(), def1)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + Expect(common.ReadYamlToObject("./testdata/log.yaml", def2)).Should(BeNil()) + Expect(k8sClient.Create(context.Background(), def2)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + }) + + It("create pipeline", func() { + var req = apisv1.CreatePipelineRequest{ + Name: pipelineName, + Description: description, + Spec: v1alpha1.WorkflowSpec{ + Steps: testPipelineSteps, + }, + } + res := post("/projects/"+projectName1+"/pipelines", req) + var pipeline apisv1.PipelineBase + Expect(decodeResponseBody(res, &pipeline)).Should(Succeed()) + Expect(cmp.Diff(pipeline.Name, req.Name)).Should(BeEmpty()) + Expect(len(pipeline.Spec.Steps)).Should(Equal(len(req.Spec.Steps))) + }) + + It("create context", func() { + var req = apisv1.CreateContextValuesRequest{ + Name: contextName, + Values: []model.Value{ + { + Key: contextKey, + Value: contextVal, + }, + }, + } + res := post("/projects/"+projectName1+"/pipelines/"+pipelineName+"/contexts", req) + var context apisv1.Context + Expect(decodeResponseBody(res, &context)).Should(Succeed()) + Expect(cmp.Diff(context.Name, req.Name)).Should(BeEmpty()) + Expect(cmp.Diff(context.Values, req.Values)).Should(BeEmpty()) + }) + + It("get contexts", func() { + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/contexts") + var contexs apisv1.ListContextValueResponse + Expect(decodeResponseBody(res, &contexs)).Should(Succeed()) + Expect(len(contexs.Contexts)).Should(Equal(1)) + ctx, ok := contexs.Contexts[contextName] + Expect(ok).Should(BeTrue()) + Expect(len(ctx)).Should(Equal(1)) + }) + + It("update context", func() { + var req = apisv1.UpdateContextValuesRequest{ + Values: []model.Value{ + { + Key: contextKey, + Value: "new-val", + }, + }, + } + res := put("/projects/"+projectName1+"/pipelines/"+pipelineName+"/contexts/"+contextName, req) + var context apisv1.Context + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + Expect(decodeResponseBody(res, &context)).Should(Succeed()) + + By("check the context value") + Expect(cmp.Diff(context.Values[0].Value, "new-val")).Should(BeEmpty()) + }) + + It("update pipeline", func() { + rawProps := []byte(`{"url":"https://api.github.com/repos/kubevela/kubevela"}`) + newSteps := make([]v1alpha1.WorkflowStep, 0) + newSteps = append(newSteps, v1alpha1.WorkflowStep{ + SubSteps: []v1alpha1.WorkflowStepBase{ + { + Name: "request1", + Type: "request", + Outputs: v1alpha1.StepOutputs{ + { + ValueFrom: "import \"strconv\"\n\"Current star count: \" + strconv.FormatInt(response[\"stargazers_count\"], 10)\n", + Name: "stars", + }, + }, + Properties: &runtime.RawExtension{ + Raw: rawProps, + }, + }, + { + Name: "request2", + Type: "request", + Outputs: v1alpha1.StepOutputs{ + { + ValueFrom: "import \"strconv\"\n\"Current star count: \" + strconv.FormatInt(response[\"stargazers_count\"], 10)\n", + Name: "stars-copy", + }, + }, + Properties: &runtime.RawExtension{ + Raw: rawProps, + }, + }, + }, + WorkflowStepBase: v1alpha1.WorkflowStepBase{ + Name: "request-group", + Type: "step-group", + }, + }) + newSteps = append(newSteps, v1alpha1.WorkflowStep{ + WorkflowStepBase: v1alpha1.WorkflowStepBase{ + Name: "log", + Type: "log", + Inputs: v1alpha1.StepInputs{ + { + ParameterKey: "data", + From: "stars", + }, + }, + }, + }) + var req = apisv1.UpdatePipelineRequest{ + Description: description, + Spec: v1alpha1.WorkflowSpec{ + Steps: newSteps, + }, + } + res := put("/projects/"+projectName1+"/pipelines/"+pipelineName, req) + var pipeline apisv1.PipelineBase + Expect(decodeResponseBody(res, &pipeline)).Should(Succeed()) + Expect(len(pipeline.Spec.Steps)).Should(Equal(len(req.Spec.Steps))) + }) + + It("run pipeline", func() { + var req = apisv1.RunPipelineRequest{ + Mode: v1alpha1.WorkflowExecuteMode{ + Steps: "StepByStep", + SubSteps: "DAG", + }, + ContextName: contextName, + } + res := post("/projects/"+projectName1+"/pipelines/"+pipelineName+"/run", req) + var run apisv1.PipelineRun + Expect(decodeResponseBody(res, &run)).Should(Succeed()) + Expect(run.PipelineRunName).ShouldNot(BeEmpty()) + pipelineRunName = run.PipelineRunName + }) + + It("list pipeline", func() { + res := get("/pipelines?query=amazing") + var pipelines apisv1.ListPipelineResponse + Expect(decodeResponseBody(res, &pipelines)).Should(Succeed()) + Expect(pipelines.Total).Should(BeNumerically("==", 1)) + Expect(pipelines.Pipelines[0].Name).Should(Equal(pipelineName)) + }) + + It("get pipeline", func() { + Eventually(func(g Gomega) { + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName) + var pipeline apisv1.GetPipelineResponse + g.Expect(decodeResponseBody(res, &pipeline)).Should(Succeed()) + g.Expect(pipeline.Name).Should(Equal(pipelineName)) + g.Expect(pipeline.Description).Should(Equal(description)) + g.Expect(pipeline.PipelineInfo.LastRun).ShouldNot(BeNil()) + g.Expect(pipeline.PipelineInfo.RunStat.Total).Should(Equal(apisv1.RunStatInfo{Total: 1, Success: 1})) + g.Expect(len(pipeline.PipelineInfo.RunStat.Week)).Should(Equal(7)) + }, 10*time.Second, 1*time.Second).Should(Succeed()) + }) + + It("list pipeline runs", func() { + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs") + var runs apisv1.ListPipelineRunResponse + Expect(decodeResponseBody(res, &runs)).Should(Succeed()) + Expect(runs.Total).Should(BeNumerically("==", 1)) + }) + + It("get pipeline run", func() { + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName) + var run apisv1.PipelineRunBase + Expect(decodeResponseBody(res, &run)).Should(Succeed()) + Expect(run.PipelineRunName).Should(Equal(pipelineRunName)) + }) + + It("get pipeline run status", func() { + Eventually(func(g Gomega) { + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName + "/status") + var status v1alpha1.WorkflowRunStatus + g.Expect(decodeResponseBody(res, &status)).Should(Succeed()) + g.Expect(status.Finished).Should(Equal(true)) + g.Expect(status.Phase).Should(Equal(v1alpha1.WorkflowStateSucceeded)) + g.Expect(status.Message).Should(BeEmpty()) + }, 100*time.Second, 1*time.Second).Should(Succeed()) + }) + + It("get pipeline run output", func() { + outputStep := "request1" + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName + "/output?step=" + outputStep) + var output apisv1.GetPipelineRunOutputResponse + Expect(decodeResponseBody(res, &output)).Should(Succeed()) + Expect(output.StepOutputs).Should(HaveLen(1)) + Expect(output.StepOutputs[0].Name).Should(Equal(outputStep)) + Expect(output.StepOutputs[0].Values).Should(HaveLen(1)) + Expect(output.StepOutputs[0].Values[0].Value).ShouldNot(BeEmpty()) + }) + + It("get pipeline run input", func() { + inputStep := "log" + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName + "/input?step=" + inputStep) + var input apisv1.GetPipelineRunInputResponse + Expect(decodeResponseBody(res, &input)).Should(Succeed()) + Expect(input.StepInputs).Should(HaveLen(1)) + Expect(input.StepInputs[0].Name).Should(Equal(inputStep)) + Expect(input.StepInputs[0].Values).Should(HaveLen(1)) + Expect(input.StepInputs[0].Values[0].Value).ShouldNot(BeEmpty()) + }) + + It("get pipeline run logs", func() { + logStep := "log" + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName + "/log?step=" + logStep) + var logs apisv1.GetPipelineRunLogResponse + Expect(decodeResponseBody(res, &logs)).Should(Succeed()) + Expect(logs.Name).Should(Equal(logStep)) + Expect(logs.Log).ShouldNot(BeEmpty()) + }) + + It("delete pipeline run", func() { + res := delete("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + }) + + It("stop pipeline", func() { + By("update pipeline so that it will run for a while") + var req = apisv1.UpdatePipelineRequest{ + Spec: v1alpha1.WorkflowSpec{ + Steps: []v1alpha1.WorkflowStep{ + { + WorkflowStepBase: v1alpha1.WorkflowStepBase{ + Name: "request", + Type: "request", + Timeout: "20s", + DependsOn: []string{"not-exist-step"}, + }, + }, + }, + }, + } + res := put("/projects/"+projectName1+"/pipelines/"+pipelineName, req) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + + By("run the pipeline") + var run apisv1.PipelineRun + res = post("/projects/"+projectName1+"/pipelines/"+pipelineName+"/run", apisv1.RunPipelineRequest{}) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + Expect(decodeResponseBody(res, &run)).Should(Succeed()) + pipelineRunName = run.PipelineRunName + + By("stop the pipeline") + var meta apisv1.PipelineRunMeta + res = post("/projects/"+projectName1+"/pipelines/"+pipelineName+"/runs/"+pipelineRunName+"/stop", nil) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + Expect(decodeResponseBody(res, &meta)).Should(Succeed()) + Expect(meta.PipelineRunName).Should(Equal(pipelineRunName)) + + By("delete pipeline run") + res = delete("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + }) + + It("delete context", func() { + res := delete("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/contexts/" + contextName) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + }) + + It("delete pipeline", func() { + res := delete("/projects/" + projectName1 + "/pipelines/" + pipelineName) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + }) + + It("delete project", func() { + res := delete("/projects/" + projectName1) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + }) + +}) diff --git a/test/e2e-apiserver-test/suite_test.go b/test/e2e-apiserver-test/suite_test.go index 4f1eb3392..1aff444b3 100644 --- a/test/e2e-apiserver-test/suite_test.go +++ b/test/e2e-apiserver-test/suite_test.go @@ -59,6 +59,7 @@ func TestE2eApiserverTest(t *testing.T) { // Suite test in e2e-apiserver-test relies on the pre-setup kubernetes environment var _ = BeforeSuite(func() { + defer GinkgoRecover() ctx := context.Background() @@ -115,7 +116,7 @@ var _ = BeforeSuite(func() { err = json.NewDecoder(resp.Body).Decode(code) Expect(err).Should(BeNil()) return fmt.Errorf("rest service not ready code:%d message:%s", resp.StatusCode, code.Message) - }, time.Second*10, time.Millisecond*200).Should(BeNil()) + }, time.Second*20, time.Millisecond*200).Should(BeNil()) var err error k8sClient, err = clients.GetKubeClient() Expect(err).ShouldNot(HaveOccurred()) diff --git a/test/e2e-apiserver-test/testdata/log.yaml b/test/e2e-apiserver-test/testdata/log.yaml new file mode 100644 index 000000000..b646796cb --- /dev/null +++ b/test/e2e-apiserver-test/testdata/log.yaml @@ -0,0 +1,35 @@ +apiVersion: core.oam.dev/v1beta1 +kind: WorkflowStepDefinition +metadata: + annotations: + definition.oam.dev/alias: "" + definition.oam.dev/description: Apply raw kubernetes objects for your workflow steps + labels: + custom.definition.oam.dev/ui-hidden: "true" + name: log + namespace: vela-system +spec: + schematic: + cue: + template: | + import ( + "vela/op" + ) + + apply: op.#Log & { + parameter + } + parameter: { + data?: string + level: *3 | int + source?: close({ + url: string + }) | close({ + resources?: [...{ + name?: string + cluster?: string + namespace?: string + labelSelector?: {...} + }] + }) + } diff --git a/test/e2e-apiserver-test/testdata/request.yaml b/test/e2e-apiserver-test/testdata/request.yaml new file mode 100644 index 000000000..2aa294d28 --- /dev/null +++ b/test/e2e-apiserver-test/testdata/request.yaml @@ -0,0 +1,43 @@ +apiVersion: core.oam.dev/v1beta1 +kind: WorkflowStepDefinition +metadata: + annotations: + definition.oam.dev/alias: "" + definition.oam.dev/description: Send request to the url + name: request + namespace: vela-system +spec: + schematic: + cue: + template: | + import ( + "vela/op" + "encoding/json" + ) + + http: op.http.#Do & { + method: parameter.method + url: parameter.url + request: { + if parameter.body != _|_ { + body: json.Marshal(parameter.body) + } + if parameter.header != _|_ { + header: parameter.header + } + } + } + fail: op.#Steps & { + if http.response.statusCode > 400 { + requestFail: op.#Fail & { + message: "request of \(parameter.url) is fail: \(http.response.statusCode)" + } + } + } + response: json.Unmarshal(http.response.body) + parameter: { + url: string + method: *"GET" | "POST" | "PUT" | "DELETE" + body?: {...} + header?: [string]: string + }