diff --git a/.github/workflows/apiserver-test.yaml b/.github/workflows/apiserver-test.yaml index 8a1661cda..b44a0eb18 100644 --- a/.github/workflows/apiserver-test.yaml +++ b/.github/workflows/apiserver-test.yaml @@ -176,10 +176,12 @@ jobs: make e2e-cleanup make e2e-setup-core bin/vela addon enable fluxcd + bin/vela addon enable vela-workflow timeout 600s bash -c -- 'while true; do kubectl get ns flux-system; if [ $? -eq 0 ] ; then break; else sleep 5; fi;done' kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-core,app.kubernetes.io/instance=kubevela -n vela-system --timeout=600s kubectl wait --for=condition=Ready pod -l app=source-controller -n flux-system --timeout=600s kubectl wait --for=condition=Ready pod -l app=helm-controller -n flux-system --timeout=600s + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-workflow -n vela-system --timeout=600s - name: Run api server e2e test run: | diff --git a/docs/apidoc/swagger.json b/docs/apidoc/swagger.json index 624f824ce..9961a4213 100644 --- a/docs/apidoc/swagger.json +++ b/docs/apidoc/swagger.json @@ -4200,900 +4200,6 @@ } } }, - "/api/v1/pipelines": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "list pipelines", - "operationId": "listPipelines", - "parameters": [ - { - "type": "string", - "description": "Fuzzy search based on name or description", - "name": "query", - "in": "query" - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.ListPipelineResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "post": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "create pipeline", - "operationId": "createPipeline", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.CreatePipelineRequest" - } - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineBase" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "get pipeline", - "operationId": "getPipeline", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.GetPipelineRequest" - } - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.GetPipelineResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "put": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "update pipeline", - "operationId": "updatePipeline", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.UpdatePipelineRequest" - } - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineBase" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "delete": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "delete pipeline", - "operationId": "deletePipeline", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineMetaResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/contexts": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "list pipeline context values", - "operationId": "listContextValues", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.ListContextValueResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "post": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "create pipeline context values", - "operationId": "createContextValue", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.CreateContextValuesRequest" - } - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.Context" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/contexts/{contextName}": { - "put": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "update pipeline context value", - "operationId": "updateContextValue", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.UpdateContextValuesRequest" - } - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline context name", - "name": "contextName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.Context" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "delete": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "delete pipeline context value", - "operationId": "deleteContextValue", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline context name", - "name": "contextName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.ContextNameResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/run": { - "post": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "run pipeline", - "operationId": "runPipeline", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1.RunPipelineRequest" - } - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineRunMeta" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "list pipeline runs", - "operationId": "listPipelineRuns", - "parameters": [ - { - "type": "string", - "description": "query identifier of the status", - "name": "status", - "in": "query" - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.ListPipelineRunResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs/{runName}": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "get pipeline run", - "operationId": "getPipelineRun", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineRunBase" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - }, - "delete": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "delete pipeline run", - "operationId": "deletePipelineRun", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineRunMeta" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs/{runName}/log": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "get pipeline run log", - "operationId": "getPipelineRunLog", - "parameters": [ - { - "type": "string", - "description": "query by specific id", - "name": "step", - "in": "query" - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.GetPipelineRunLogResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs/{runName}/output": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "get pipeline run output", - "operationId": "getPipelineRunOutput", - "parameters": [ - { - "type": "string", - "description": "query by specific id", - "name": "step", - "in": "query" - }, - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.GetPipelineRunOutputResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs/{runName}/status": { - "get": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "get pipeline run status", - "operationId": "getPipelineRunStatus", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1alpha1.WorkflowRunStatus" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, - "/api/v1/pipelines/{pipelineName}/runs/{runName}/stop": { - "post": { - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "tags": [ - "pipeline" - ], - "summary": "stop pipeline run", - "operationId": "stopPipeline", - "parameters": [ - { - "type": "string", - "description": "project name", - "name": "projectName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "pipeline name", - "name": "pipelineName", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "pipeline run name", - "name": "runName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/v1.PipelineRunMeta" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/bcode.Bcode" - } - } - } - } - }, "/api/v1/projects": { "get": { "consumes": [ @@ -5152,6 +4258,45 @@ } } }, + "/api/v1/projects/pipelines": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "list pipelines", + "operationId": "listPipelines", + "parameters": [ + { + "type": "string", + "description": "Fuzzy search based on name or description", + "name": "query", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.ListPipelineResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, "/api/v1/projects/{projectName}": { "get": { "consumes": [ @@ -5845,6 +4990,863 @@ } } }, + "/api/v1/projects/{projectName}/pipelines": { + "post": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "create pipeline", + "operationId": "createPipeline", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.CreatePipelineRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineBase" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "get pipeline", + "operationId": "getPipeline", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.GetPipelineRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.GetPipelineResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + }, + "put": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "update pipeline", + "operationId": "updatePipeline", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.UpdatePipelineRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineBase" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + }, + "delete": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "delete pipeline", + "operationId": "deletePipeline", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineMetaResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/contexts": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "list pipeline context values", + "operationId": "listContextValues", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.ListContextValueResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + }, + "post": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "create pipeline context values", + "operationId": "createContextValue", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.CreateContextValuesRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.Context" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/contexts/{contextName}": { + "put": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "update pipeline context value", + "operationId": "updateContextValue", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.UpdateContextValuesRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline context name", + "name": "contextName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.Context" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + }, + "delete": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "delete pipeline context value", + "operationId": "deleteContextValue", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline context name", + "name": "contextName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.ContextNameResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/run": { + "post": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "run pipeline", + "operationId": "runPipeline", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.RunPipelineRequest" + } + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineRunMeta" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "list pipeline runs", + "operationId": "listPipelineRuns", + "parameters": [ + { + "type": "string", + "description": "query identifier of the status", + "name": "status", + "in": "query" + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.ListPipelineRunResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs/{runName}": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "get pipeline run", + "operationId": "getPipelineRun", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineRunBase" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + }, + "delete": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "delete pipeline run", + "operationId": "deletePipelineRun", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineRunMeta" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs/{runName}/log": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "get pipeline run log", + "operationId": "getPipelineRunLog", + "parameters": [ + { + "type": "string", + "description": "query by specific step name", + "name": "step", + "in": "query" + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.GetPipelineRunLogResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs/{runName}/output": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "get pipeline run output", + "operationId": "getPipelineRunOutput", + "parameters": [ + { + "type": "string", + "description": "query by specific id", + "name": "step", + "in": "query" + }, + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.GetPipelineRunOutputResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs/{runName}/status": { + "get": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "get pipeline run status", + "operationId": "getPipelineRunStatus", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1alpha1.WorkflowRunStatus" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, + "/api/v1/projects/{projectName}/pipelines/{pipelineName}/runs/{runName}/stop": { + "post": { + "consumes": [ + "application/xml", + "application/json" + ], + "produces": [ + "application/json", + "application/xml" + ], + "tags": [ + "pipeline" + ], + "summary": "stop pipeline run", + "operationId": "stopPipeline", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "projectName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline name", + "name": "pipelineName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "pipeline run name", + "name": "runName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.PipelineRunMeta" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/bcode.Bcode" + } + } + } + } + }, "/api/v1/projects/{projectName}/providers": { "get": { "consumes": [ @@ -7355,6 +7357,12 @@ "in": "path", "required": true }, + { + "type": "string", + "description": "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + "name": "dryRun", + "in": "query" + }, { "name": "body", "in": "body", @@ -8358,8 +8366,8 @@ }, "config.ClusterTargetStatus": { "required": [ - "namespace", "clusterName", + "namespace", "status", "application", "message" @@ -8648,8 +8656,8 @@ }, "model.Cluster": { "required": [ - "createTime", "updateTime", + "createTime", "name", "alias", "description", @@ -9444,12 +9452,12 @@ }, "v1.ApplicationDeployResponse": { "required": [ - "status", "note", - "createTime", - "envName", "triggerType", - "version" + "createTime", + "version", + "status", + "envName" ], "properties": { "codeInfo": { @@ -10209,13 +10217,13 @@ }, "v1.ConfigTemplateDetail": { "required": [ + "alias", + "name", "namespace", "description", "scope", "sensitive", "createTime", - "alias", - "name", "schema", "uiSchema" ], @@ -11035,11 +11043,11 @@ }, "v1.DetailAddonResponse": { "required": [ + "invisible", + "name", "version", "description", "icon", - "name", - "invisible", "schema", "uiSchema", "definitions", @@ -11119,13 +11127,13 @@ }, "v1.DetailApplicationResponse": { "required": [ - "updateTime", - "name", "alias", - "project", "description", - "createTime", "icon", + "name", + "project", + "createTime", + "updateTime", "policies", "envBindings", "resourceInfo" @@ -11182,20 +11190,20 @@ }, "v1.DetailClusterResponse": { "required": [ - "updateTime", - "name", "apiServerURL", - "kubeConfigSecret", - "createTime", "alias", - "provider", - "kubeConfig", "description", "icon", - "reason", + "kubeConfigSecret", + "name", "labels", - "status", + "reason", + "provider", "dashboardURL", + "updateTime", + "createTime", + "status", + "kubeConfig", "resourceInfo" ], "properties": { @@ -11253,14 +11261,14 @@ }, "v1.DetailComponentResponse": { "required": [ - "appPrimaryKey", + "updateTime", + "creator", + "alias", "main", "createTime", "type", "name", - "alias", - "updateTime", - "creator", + "appPrimaryKey", "definition" ], "properties": { @@ -11348,13 +11356,13 @@ }, "v1.DetailDefinitionResponse": { "required": [ + "alias", "description", "status", - "ownerAddon", - "name", - "alias", - "icon", "labels", + "name", + "icon", + "ownerAddon", "schema", "uiSchema" ], @@ -11411,15 +11419,15 @@ }, "v1.DetailPolicyResponse": { "required": [ - "name", + "envName", "alias", + "description", "creator", "properties", "createTime", - "updateTime", - "envName", + "name", "type", - "description" + "updateTime" ], "properties": { "alias": { @@ -11461,18 +11469,18 @@ }, "v1.DetailRevisionResponse": { "required": [ - "version", + "triggerType", + "note", "revisionCRName", - "workflowName", - "reason", - "updateTime", + "status", "appPrimaryKey", + "version", "envName", "createTime", - "status", + "reason", "deployUser", - "note", - "triggerType" + "workflowName", + "updateTime" ], "properties": { "appPrimaryKey": { @@ -11529,8 +11537,8 @@ }, "v1.DetailTargetResponse": { "required": [ - "updateTime", "project", + "updateTime", "name", "createTime" ], @@ -11572,11 +11580,11 @@ }, "v1.DetailUserResponse": { "required": [ - "email", - "disabled", "createTime", "lastLoginTime", "name", + "email", + "disabled", "projects", "roles" ], @@ -11674,13 +11682,13 @@ }, "v1.DetailWorkflowResponse": { "required": [ + "createTime", + "name", "alias", "enable", - "envName", - "name", - "description", "default", - "createTime", + "description", + "envName", "updateTime" ], "properties": { @@ -11913,8 +11921,8 @@ }, "v1.GetPipelineResponse": { "required": [ - "project", "name", + "project", "spec", "info" ], @@ -11941,14 +11949,27 @@ }, "v1.GetPipelineRunLogResponse": { "required": [ + "phase", + "id", + "name", + "type", "log" ], "properties": { + "id": { + "type": "string" + }, "log": { - "type": "array", - "items": { - "$ref": "#/definitions/v1.Log" - } + "type": "string" + }, + "name": { + "type": "string" + }, + "phase": { + "type": "string" + }, + "type": { + "type": "string" } } }, @@ -11960,7 +11981,7 @@ "output": { "type": "array", "items": { - "$ref": "#/definitions/v1.Output" + "$ref": "#/definitions/v1.StepOutput" } } } @@ -12526,32 +12547,6 @@ } } }, - "v1.Log": { - "required": [ - "id", - "name", - "type", - "phase", - "log" - ], - "properties": { - "id": { - "type": "string" - }, - "log": { - "type": "string" - }, - "name": { - "type": "string" - }, - "phase": { - "type": "string" - }, - "type": { - "type": "string" - } - } - }, "v1.LoginRequest": { "properties": { "code": { @@ -12729,35 +12724,6 @@ } } }, - "v1.Output": { - "required": [ - "id", - "name", - "type", - "phase", - "vars" - ], - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "phase": { - "type": "string" - }, - "type": { - "type": "string" - }, - "vars": { - "type": "object", - "additionalProperties": { - "type": "string" - } - } - } - }, "v1.PermissionBase": { "required": [ "name", @@ -12802,8 +12768,8 @@ }, "v1.PipelineBase": { "required": [ - "project", "name", + "project", "spec" ], "properties": { @@ -12891,8 +12857,8 @@ }, "v1.PipelineMetaResponse": { "required": [ - "name", - "project" + "project", + "name" ], "properties": { "alias": { @@ -12911,9 +12877,9 @@ }, "v1.PipelineRunBase": { "required": [ - "project", "pipelineRunName", "pipelineName", + "project", "record", "contextName", "spec" @@ -13331,6 +13297,52 @@ } } }, + "v1.StepOutput": { + "required": [ + "outputs", + "subStepOutputs" + ], + "properties": { + "outputs": { + "$ref": "#/definitions/v1.StepOutputBase" + }, + "subStepOutputs": { + "type": "array", + "items": { + "$ref": "#/definitions/v1.StepOutputBase" + } + } + } + }, + "v1.StepOutputBase": { + "required": [ + "phase", + "id", + "name", + "type", + "vars" + ], + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "phase": { + "type": "string" + }, + "type": { + "type": "string" + }, + "vars": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, "v1.SystemInfo": { "required": [ "platformID", @@ -13390,9 +13402,9 @@ }, "v1.SystemInfoResponse": { "required": [ + "enableCollection", "loginType", "platformID", - "enableCollection", "systemVersion" ], "properties": { @@ -14022,6 +14034,9 @@ }, "v1alpha1.WorkflowRunSpec": { "properties": { + "context": { + "type": "string" + }, "mode": { "$ref": "#/definitions/v1alpha1.WorkflowExecuteMode" }, diff --git a/e2e/addon/mock/testrepo/helm-repo/index.yaml b/e2e/addon/mock/testrepo/helm-repo/index.yaml index e496f9609..8d5bc3211 100644 --- a/e2e/addon/mock/testrepo/helm-repo/index.yaml +++ b/e2e/addon/mock/testrepo/helm-repo/index.yaml @@ -23,4 +23,15 @@ entries: annotations: system.vela: ">=1.5.0" system.kubernetes: ">=1.30.0" + vela-workflow: + - annotations: + system.vela: '>=v1.6.0-beta.1' + created: "2022-10-29T09:11:16.865230605Z" + description: vela-workflow provides the capability to run a standalone workflow + home: https://github.com/kubevela/workflow + icon: https://static.kubevela.net/images/logos/KubeVela%20-03.png + name: vela-workflow + urls: + - http://127.0.0.1:9098/helm/vela-workflow-v0.3.1.tgz + version: v0.3.1 generated: "2022-06-15T13:17:04.733573+08:00" \ No newline at end of file diff --git a/e2e/addon/mock/testrepo/helm-repo/vela-workflow-v0.3.1.tgz b/e2e/addon/mock/testrepo/helm-repo/vela-workflow-v0.3.1.tgz new file mode 100644 index 000000000..94495cb85 Binary files /dev/null and b/e2e/addon/mock/testrepo/helm-repo/vela-workflow-v0.3.1.tgz differ diff --git a/e2e/addon/mock/vela_addon_mock_server.go b/e2e/addon/mock/vela_addon_mock_server.go index 98faa56dd..2808a4c69 100644 --- a/e2e/addon/mock/vela_addon_mock_server.go +++ b/e2e/addon/mock/vela_addon_mock_server.go @@ -22,9 +22,9 @@ import ( "fmt" "html/template" "io/fs" - "io/ioutil" "log" "net/http" + "os" "path" "strings" @@ -108,24 +108,31 @@ var ossHandler http.HandlerFunc = func(rw http.ResponseWriter, req *http.Request var helmHandler http.HandlerFunc = func(rw http.ResponseWriter, req *http.Request) { switch { case strings.Contains(req.URL.Path, "index.yaml"): - file, err := ioutil.ReadFile("./e2e/addon/mock/testrepo/helm-repo/index.yaml") + file, err := os.ReadFile("./e2e/addon/mock/testrepo/helm-repo/index.yaml") if err != nil { _, _ = rw.Write([]byte(err.Error())) } rw.Write(file) case strings.Contains(req.URL.Path, "fluxcd-test-version-1.0.0.tgz"): - file, err := ioutil.ReadFile("./e2e/addon/mock/testrepo/helm-repo/fluxcd-test-version-1.0.0.tgz") + file, err := os.ReadFile("./e2e/addon/mock/testrepo/helm-repo/fluxcd-test-version-1.0.0.tgz") if err != nil { _, _ = rw.Write([]byte(err.Error())) } rw.Write(file) case strings.Contains(req.URL.Path, "fluxcd-test-version-2.0.0.tgz"): - file, err := ioutil.ReadFile("./e2e/addon/mock/testrepo/helm-repo/fluxcd-test-version-2.0.0.tgz") + file, err := os.ReadFile("./e2e/addon/mock/testrepo/helm-repo/fluxcd-test-version-2.0.0.tgz") + if err != nil { + _, _ = rw.Write([]byte(err.Error())) + } + rw.Write(file) + case strings.Contains(req.URL.Path, "vela-workflow-v0.3.1.tgz"): + file, err := os.ReadFile("./e2e/addon/mock/testrepo/helm-repo/vela-workflow-v0.3.1.tgz") if err != nil { _, _ = rw.Write([]byte(err.Error())) } rw.Write(file) } + } func init() { diff --git a/go.mod b/go.mod index cbf271da4..052f040de 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( github.com/kubevela/workflow v0.3.1 github.com/kyokomi/emoji v2.2.4+incompatible github.com/mitchellh/hashstructure/v2 v2.0.1 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd github.com/oam-dev/cluster-gateway v1.4.0 github.com/oam-dev/cluster-register v1.0.4-0.20220928064144-5f76a9d7ca8c github.com/oam-dev/terraform-config-inspect v0.0.0-20210418082552-fc72d929aa28 @@ -239,7 +240,6 @@ require ( github.com/moby/locker v1.0.1 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/morikuni/aec v1.0.0 // indirect diff --git a/pkg/addon/addon.go b/pkg/addon/addon.go index 0dbdc6dec..e422546ed 100644 --- a/pkg/addon/addon.go +++ b/pkg/addon/addon.go @@ -612,7 +612,7 @@ func unmarshalToContent(content []byte) (fileContent *github.RepositoryContent, } func genAddonAPISchema(addonRes *UIData) error { - cueScript := script.CUE([]byte(addonRes.Parameters)) + cueScript := script.CUE(addonRes.Parameters) schema, err := cueScript.ParsePropertiesToSchema() if err != nil { return err diff --git a/pkg/apiserver/domain/model/context.go b/pkg/apiserver/domain/model/pipeline.go similarity index 62% rename from pkg/apiserver/domain/model/context.go rename to pkg/apiserver/domain/model/pipeline.go index 423426dcd..eac2dd877 100644 --- a/pkg/apiserver/domain/model/context.go +++ b/pkg/apiserver/domain/model/pipeline.go @@ -16,10 +16,52 @@ limitations under the License. package model -import "fmt" +import ( + "fmt" + + "github.com/kubevela/workflow/api/v1alpha1" +) func init() { RegisterModel(&PipelineContext{}) + RegisterModel(&Pipeline{}) +} + +// Pipeline is the model of pipeline +type Pipeline struct { + BaseModel + Spec v1alpha1.WorkflowSpec + Name string `json:"name"` + Project string `json:"project"` + Alias string `json:"alias"` + Description string `json:"description"` +} + +// PrimaryKey return custom primary key +func (p Pipeline) PrimaryKey() string { + return fmt.Sprintf("%s-%s", p.Project, p.Name) +} + +// TableName return custom table name +func (p Pipeline) TableName() string { + return tableNamePrefix + "pipeline" +} + +// ShortTableName is the compressed version of table name for kubeapi storage and others +func (p Pipeline) ShortTableName() string { + return "pipeline" +} + +// Index return custom index +func (p Pipeline) Index() map[string]string { + var index = make(map[string]string) + if p.Project != "" { + index["project"] = p.Project + } + if p.Name != "" { + index["name"] = p.Name + } + return index } // Value is a k-v pair diff --git a/pkg/apiserver/domain/model/project.go b/pkg/apiserver/domain/model/project.go index 6911b06b8..912c1cfa6 100644 --- a/pkg/apiserver/domain/model/project.go +++ b/pkg/apiserver/domain/model/project.go @@ -16,8 +16,6 @@ limitations under the License. package model -import "fmt" - func init() { RegisterModel(&Project{}) } @@ -29,11 +27,15 @@ type Project struct { Alias string `json:"alias"` Owner string `json:"owner"` Description string `json:"description,omitempty"` + Namespace string `json:"namespace"` } // GetNamespace get the namespace name of this project. func (p *Project) GetNamespace() string { - return fmt.Sprintf("project-%s", p.Name) + if p.Namespace != "" { + return p.Namespace + } + return p.Name } // TableName return custom table name diff --git a/pkg/apiserver/domain/service/cloudshell_test.go b/pkg/apiserver/domain/service/cloudshell_test.go index 5fd90527f..c5daf7e8b 100644 --- a/pkg/apiserver/domain/service/cloudshell_test.go +++ b/pkg/apiserver/domain/service/cloudshell_test.go @@ -66,7 +66,8 @@ var _ = Describe("Test cloudshell service function", func() { ProjectService: projectService, } projectService = &projectServiceImpl{ - Store: ds, + Store: ds, + K8sClient: k8sClient, RbacService: &rbacServiceImpl{ Store: ds, }, diff --git a/pkg/apiserver/domain/service/helm_test.go b/pkg/apiserver/domain/service/helm_test.go index c5076cf13..762d27413 100644 --- a/pkg/apiserver/domain/service/helm_test.go +++ b/pkg/apiserver/domain/service/helm_test.go @@ -83,7 +83,7 @@ var _ = Describe("Test helm repo list", func() { pSec = v1.Secret{} gSec = v1.Secret{} Expect(k8sClient.Create(ctx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "vela-system"}})).Should(SatisfyAny(BeNil(), util.AlreadyExistMatcher{})) - Expect(k8sClient.Create(ctx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "project-my-project"}})).Should(SatisfyAny(BeNil(), util.AlreadyExistMatcher{})) + Expect(k8sClient.Create(ctx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "my-project"}})).Should(SatisfyAny(BeNil(), util.AlreadyExistMatcher{})) Expect(yaml.Unmarshal([]byte(projectSecret), &pSec)).Should(BeNil()) Expect(yaml.Unmarshal([]byte(globalSecret), &gSec)).Should(BeNil()) Expect(k8sClient.Create(ctx, &pSec)).Should(BeNil()) @@ -390,7 +390,7 @@ apiVersion: v1 kind: Secret metadata: name: project-helm-repo - namespace: project-my-project + namespace: my-project labels: config.oam.dev/type: helm-repository config.oam.dev/catalog: velacore-config diff --git a/pkg/apiserver/domain/service/pipeline.go b/pkg/apiserver/domain/service/pipeline.go index 8aff1049d..183f68bc9 100644 --- a/pkg/apiserver/domain/service/pipeline.go +++ b/pkg/apiserver/domain/service/pipeline.go @@ -17,76 +17,97 @@ limitations under the License. package service import ( + "bufio" + "bytes" "context" "fmt" + "hash/fnv" + "io" + "sort" "strings" + "sync" + "time" + "github.com/fatih/color" + + "github.com/kubevela/workflow/api/v1alpha1" + "github.com/kubevela/workflow/pkg/cue/model/value" + wfTypes "github.com/kubevela/workflow/pkg/types" + wfUtils "github.com/kubevela/workflow/pkg/utils" + "github.com/modern-go/concurrent" "github.com/pkg/errors" - + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/kubevela/workflow/api/v1alpha1" - wfTypes "github.com/kubevela/workflow/pkg/types" - + types2 "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/apiserver/domain/model" "github.com/oam-dev/kubevela/pkg/apiserver/infrastructure/datastore" apis "github.com/oam-dev/kubevela/pkg/apiserver/interfaces/api/dto/v1" "github.com/oam-dev/kubevela/pkg/apiserver/utils" "github.com/oam-dev/kubevela/pkg/apiserver/utils/bcode" "github.com/oam-dev/kubevela/pkg/apiserver/utils/log" - "github.com/oam-dev/kubevela/pkg/utils/apply" + "github.com/oam-dev/kubevela/pkg/oam/util" + pkgutils "github.com/oam-dev/kubevela/pkg/utils" ) const ( - labelDescription = "pipeline.velaux.oam.dev/description" - labelAlias = "pipeline.velaux.oam.dev/alias" -) - -const ( - labelContextName = "context.velaux.oam.dev/name" + labelContext = "pipeline.oam.dev/context" + labelPipeline = "pipeline.oam.dev/name" ) // PipelineService is the interface for pipeline service type PipelineService interface { CreatePipeline(ctx context.Context, req apis.CreatePipelineRequest) (*apis.PipelineBase, error) ListPipelines(ctx context.Context, req apis.ListPipelineRequest) (*apis.ListPipelineResponse, error) - GetPipeline(ctx context.Context, name, project string) (*apis.GetPipelineResponse, error) - UpdatePipeline(ctx context.Context, name, project string, req apis.UpdatePipelineRequest) (*apis.PipelineBase, error) + GetPipeline(ctx context.Context, name string, getInfo bool) (*apis.GetPipelineResponse, error) + UpdatePipeline(ctx context.Context, name string, req apis.UpdatePipelineRequest) (*apis.PipelineBase, error) DeletePipeline(ctx context.Context, base apis.PipelineBase) error - RunPipeline(ctx context.Context, pipeline apis.PipelineBase, req apis.RunPipelineRequest) error + RunPipeline(ctx context.Context, pipeline apis.PipelineBase, req apis.RunPipelineRequest) (*apis.PipelineRun, error) } type pipelineServiceImpl struct { - ContextService ContextService `inject:""` - KubeClient client.Client `inject:"kubeClient"` - KubeConfig *rest.Config `inject:"kubeConfig"` - Apply apply.Applicator `inject:"apply"` + Store datastore.DataStore `inject:"datastore"` + ProjectService ProjectService `inject:""` + ContextService ContextService `inject:""` + KubeClient client.Client `inject:"kubeClient"` + KubeConfig *rest.Config `inject:"kubeConfig"` + PipelineRunService PipelineRunService `inject:""` } // PipelineRunService is the interface for pipelineRun service type PipelineRunService interface { - GetPipelineRun(ctx context.Context, meta apis.PipelineRunMeta) (apis.PipelineRun, error) + GetPipelineRun(ctx context.Context, meta apis.PipelineRunMeta) (*apis.PipelineRun, error) ListPipelineRuns(ctx context.Context, base apis.PipelineBase) (apis.ListPipelineRunResponse, error) DeletePipelineRun(ctx context.Context, meta apis.PipelineRunMeta) error + CleanPipelineRuns(ctx context.Context, base apis.PipelineBase) error StopPipelineRun(ctx context.Context, pipeline apis.PipelineRunBase) error + GetPipelineRunOutput(ctx context.Context, meta apis.PipelineRun, step string) (apis.GetPipelineRunOutputResponse, error) + GetPipelineRunInput(ctx context.Context, meta apis.PipelineRun, step string) (apis.GetPipelineRunInputResponse, error) + GetPipelineRunLog(ctx context.Context, meta apis.PipelineRun, step string) (apis.GetPipelineRunLogResponse, error) } type pipelineRunServiceImpl struct { - KubeClient client.Client `inject:"kubeClient"` - KubeConfig *rest.Config `inject:"kubeConfig"` - Apply apply.Applicator `inject:"apply"` - ContextService ContextService `inject:""` + Store datastore.DataStore `inject:"datastore"` + KubeClient client.Client `inject:"kubeClient"` + KubeConfig *rest.Config `inject:"kubeConfig"` + ContextService ContextService `inject:""` + ProjectService ProjectService `inject:""` } // ContextService is the interface for context service type ContextService interface { + InitContext(ctx context.Context, projectName, pipelineName string) (*model.PipelineContext, error) GetContext(ctx context.Context, projectName, pipelineName string, name string) (*apis.Context, error) CreateContext(ctx context.Context, projectName, pipelineName string, context apis.Context) (*model.PipelineContext, error) UpdateContext(ctx context.Context, projectName, pipelineName string, context apis.Context) (*model.PipelineContext, error) ListContexts(ctx context.Context, projectName, pipelineName string) (*apis.ListContextValueResponse, error) DeleteContext(ctx context.Context, projectName, pipelineName, name string) error + DeleteAllContexts(ctx context.Context, projectName, pipelineName string) error } type contextServiceImpl struct { @@ -110,90 +131,196 @@ func NewContextService() ContextService { // CreatePipeline will create a pipeline func (p pipelineServiceImpl) CreatePipeline(ctx context.Context, req apis.CreatePipelineRequest) (*apis.PipelineBase, error) { - wf := v1alpha1.Workflow{} - wf.SetName(req.Name) - wf.SetNamespace(nsForProj(req.Project)) - wf.WorkflowSpec = req.Spec - wf.SetLabels(map[string]string{ - labelDescription: req.Description, - labelAlias: req.Alias}) - if err := p.KubeClient.Create(ctx, &wf); err != nil { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) + if err := checkPipelineSpec(req.Spec); err != nil { + return nil, err + } + pipeline := &model.Pipeline{ + Name: req.Name, + Description: req.Description, + Alias: req.Alias, + Project: project.Name, + Spec: req.Spec, + } + if err := p.Store.Add(ctx, pipeline); err != nil { + if errors.Is(err, datastore.ErrRecordExist) { + return nil, bcode.ErrPipelineExist + } return nil, err } return &apis.PipelineBase{ PipelineMeta: apis.PipelineMeta{ - Name: req.Name, - Alias: req.Alias, - Project: req.Project, + Name: req.Name, + Alias: req.Alias, + Project: apis.NameAlias{ + Name: project.Name, + Alias: project.Alias, + }, Description: req.Description, }, - Spec: wf.WorkflowSpec, + Spec: pipeline.Spec, }, nil } // ListPipelines will list all pipelines func (p pipelineServiceImpl) ListPipelines(ctx context.Context, req apis.ListPipelineRequest) (*apis.ListPipelineResponse, error) { - wfs := v1alpha1.WorkflowList{} - nsOption := make([]client.ListOption, 0) - for _, ns := range req.Projects { - nsOption = append(nsOption, client.InNamespace(nsForProj(ns))) + userName, ok := ctx.Value(&apis.CtxKeyUser).(string) + if !ok { + return nil, bcode.ErrUnauthorized } - if err := p.KubeClient.List(ctx, &wfs, nsOption...); err != nil { + projects, err := p.ProjectService.ListUserProjects(ctx, userName) + if err != nil { + return nil, err + } + var availableProjectNames []string + var projectMap = make(map[string]model.Project, len(projects)) + for _, project := range projects { + availableProjectNames = append(availableProjectNames, project.Name) + // We only need name, namespace and alias of project + projectMap[project.Name] = model.Project{Name: project.Name, Alias: project.Alias, Namespace: project.Namespace} + + } + if len(availableProjectNames) == 0 { + return &apis.ListPipelineResponse{}, nil + } + pp := model.Pipeline{} + pipelines, err := p.Store.List(ctx, &pp, &datastore.ListOptions{ + FilterOptions: datastore.FilterOptions{ + In: []datastore.InQueryOption{ + { + Key: "project", + Values: availableProjectNames, + }, + }, + }, + }) + if err != nil { return nil, err } res := apis.ListPipelineResponse{} - for _, wf := range wfs.Items { - if fuzzyMatch(wf, req.Query) { + for _, _p := range pipelines { + pipeline := _p.(*model.Pipeline) + if !pkgutils.StringsContain(availableProjectNames, pipeline.Project) { + continue + } + if fuzzyMatch(pipeline, req.Query) { + base := pipeline2PipelineBase(pipeline, projectMap[pipeline.Project]) + var info *apis.PipelineInfo + if req.Detailed { + info, err = p.getPipelineInfo(ctx, pipeline, projectMap[pipeline.Project].Namespace) + if err != nil { + // Since we are listing pipelines. We should not return directly if we cannot get pipeline info + log.Logger.Errorf("get pipeline %s/%s info error: %v", pipeline.Project, pipeline.Name, err) + continue + } + } item := apis.PipelineListItem{ - PipelineMeta: workflow2PipelineBase(wf).PipelineMeta, - // todo info + PipelineMeta: base.PipelineMeta, + Info: func() apis.PipelineInfo { + if info != nil { + return *info + } + return apis.PipelineInfo{} + }(), } res.Pipelines = append(res.Pipelines, item) } } + res.Total = len(res.Pipelines) return &res, nil } // GetPipeline will get a pipeline -func (p pipelineServiceImpl) GetPipeline(ctx context.Context, name, project string) (*apis.GetPipelineResponse, error) { - wf := v1alpha1.Workflow{} - if err := p.KubeClient.Get(ctx, client.ObjectKey{Name: name, Namespace: nsForProj(project)}, &wf); err != nil { +func (p pipelineServiceImpl) GetPipeline(ctx context.Context, name string, getInfo bool) (*apis.GetPipelineResponse, error) { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) + pipeline := &model.Pipeline{ + Name: name, + Project: project.Name, + } + if err := p.Store.Get(ctx, pipeline); err != nil { + if errors.Is(err, datastore.ErrRecordNotExist) { + return nil, bcode.ErrPipelineNotExist + } return nil, err } + base := pipeline2PipelineBase(pipeline, *project) + var info = apis.PipelineInfo{} + if getInfo { + in, err := p.getPipelineInfo(ctx, pipeline, project.Namespace) + if err != nil { + log.Logger.Errorf("get pipeline %s/%s info error: %v", pipeline.Project, pipeline.Name, err) + return nil, bcode.ErrGetPipelineInfo + } + if in != nil { + info = *in + } + } + return &apis.GetPipelineResponse{ - PipelineBase: *workflow2PipelineBase(wf), - // todo info + PipelineBase: *base, + PipelineInfo: info, }, nil } // UpdatePipeline will update a pipeline -func (p pipelineServiceImpl) UpdatePipeline(ctx context.Context, name, project string, req apis.UpdatePipelineRequest) (*apis.PipelineBase, error) { - wf := v1alpha1.Workflow{} - if err := p.KubeClient.Get(ctx, client.ObjectKey{Name: name, Namespace: nsForProj(project)}, &wf); err != nil { +func (p pipelineServiceImpl) UpdatePipeline(ctx context.Context, name string, req apis.UpdatePipelineRequest) (*apis.PipelineBase, error) { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) + if err := checkPipelineSpec(req.Spec); err != nil { return nil, err } - wf.WorkflowSpec = req.Spec - wf.SetLabels(map[string]string{ - labelDescription: req.Description, - labelAlias: req.Alias}) - if err := p.KubeClient.Update(ctx, &wf); err != nil { + pipeline := &model.Pipeline{ + Name: name, + Project: project.Name, + } + if err := p.Store.Get(ctx, pipeline); err != nil { + if errors.Is(err, datastore.ErrRecordNotExist) { + return nil, bcode.ErrPipelineNotExist + } return nil, err } - return workflow2PipelineBase(wf), nil + + pipeline.Spec = req.Spec + pipeline.Description = req.Description + pipeline.Alias = req.Alias + + if err := p.Store.Put(ctx, pipeline); err != nil { + return nil, err + } + return pipeline2PipelineBase(pipeline, *project), nil } // DeletePipeline will delete a pipeline func (p pipelineServiceImpl) DeletePipeline(ctx context.Context, pl apis.PipelineBase) error { - wf := v1alpha1.Workflow{} - if err := p.KubeClient.Get(ctx, client.ObjectKey{Name: pl.Name, Namespace: nsForProj(pl.Project)}, &wf); err != nil { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) + pipeline := &model.Pipeline{ + Name: pl.Name, + Project: project.Name, + } + if err := p.Store.Get(ctx, pipeline); err != nil { + if errors.Is(err, datastore.ErrRecordNotExist) { + return bcode.ErrPipelineNotExist + } return err } - return p.KubeClient.Delete(ctx, &wf) + // Clean up pipeline: 1. delete pipeline runs 2. delete contexts 3. delete pipeline + if err := p.PipelineRunService.CleanPipelineRuns(ctx, pl); err != nil { + log.Logger.Errorf("delete pipeline all pipeline-runs failure: %s", err.Error()) + return err + } + if err := p.ContextService.DeleteAllContexts(ctx, pl.Project.Name, pl.Name); err != nil { + log.Logger.Errorf("delete pipeline all context failure: %s", err.Error()) + return err + } + if err := p.Store.Delete(ctx, pipeline); err != nil { + return err + } + + return nil } // StopPipelineRun will stop a pipelineRun func (p pipelineRunServiceImpl) StopPipelineRun(ctx context.Context, pipelineRun apis.PipelineRunBase) error { - run, err := p.checkRecordRunning(ctx, pipelineRun) + run, err := p.checkRunNotFinished(ctx, pipelineRun) if err != nil { return err } @@ -203,53 +330,625 @@ func (p pipelineRunServiceImpl) StopPipelineRun(ctx context.Context, pipelineRun return nil } +func (p pipelineRunServiceImpl) GetPipelineRunOutput(ctx context.Context, pipelineRun apis.PipelineRun, stepName string) (apis.GetPipelineRunOutputResponse, error) { + outputsSpec := make(map[string]v1alpha1.StepOutputs) + stepOutputs := make([]apis.StepOutputBase, 0) + if pipelineRun.Spec.WorkflowSpec != nil { + for _, step := range pipelineRun.Spec.WorkflowSpec.Steps { + if step.Outputs != nil { + outputsSpec[step.Name] = step.Outputs + } + for _, sub := range step.SubSteps { + if sub.Outputs != nil { + outputsSpec[sub.Name] = sub.Outputs + } + } + } + } + ctxBackend := pipelineRun.Status.ContextBackend + if ctxBackend == nil { + return apis.GetPipelineRunOutputResponse{}, nil + } + v, err := wfUtils.GetDataFromContext(ctx, p.KubeClient, ctxBackend.Name, pipelineRun.PipelineRunName, ctxBackend.Namespace) + if err != nil { + log.Logger.Errorf("get data from context backend failed: %v", err) + return apis.GetPipelineRunOutputResponse{}, bcode.ErrGetContextBackendData + } + for _, s := range pipelineRun.Status.Steps { + var ( + subStepStatus *v1alpha1.StepStatus + ok bool + ) + if stepName != "" && s.Name != stepName { + subStepStatus, ok = haveSubSteps(s, stepName) + if !ok { + continue + } + subVars := getStepOutputs(*subStepStatus, outputsSpec, v) + stepOutputs = append(stepOutputs, subVars) + break + } + stepOutputs = append(stepOutputs, getStepOutputs(s.StepStatus, outputsSpec, v)) + for _, sub := range s.SubStepsStatus { + stepOutputs = append(stepOutputs, getStepOutputs(sub, outputsSpec, v)) + } + if stepName != "" && s.Name == stepName { + // already found the step + break + } + } + return apis.GetPipelineRunOutputResponse{StepOutputs: stepOutputs}, nil +} + +func (p pipelineRunServiceImpl) GetPipelineRunInput(ctx context.Context, pipelineRun apis.PipelineRun, stepName string) (apis.GetPipelineRunInputResponse, error) { + // valueFromStep know which step the value came from + valueFromStep := make(map[string]string) + inputsSpec := make(map[string]v1alpha1.StepInputs) + stepInputs := make([]apis.StepInputBase, 0) + if pipelineRun.Spec.WorkflowSpec != nil { + for _, step := range pipelineRun.Spec.WorkflowSpec.Steps { + if step.Inputs != nil { + inputsSpec[step.Name] = step.Inputs + } + if step.Outputs != nil { + for _, o := range step.Outputs { + valueFromStep[o.Name] = step.Name + } + } + for _, sub := range step.SubSteps { + if sub.Inputs != nil { + inputsSpec[sub.Name] = sub.Inputs + } + if sub.Outputs != nil { + for _, o := range sub.Outputs { + valueFromStep[o.Name] = sub.Name + } + } + } + } + } + ctxBackend := pipelineRun.Status.ContextBackend + if ctxBackend == nil { + return apis.GetPipelineRunInputResponse{}, nil + } + v, err := wfUtils.GetDataFromContext(ctx, p.KubeClient, ctxBackend.Name, pipelineRun.PipelineRunName, ctxBackend.Namespace) + if err != nil { + log.Logger.Errorf("get data from context backend failed: %v", err) + return apis.GetPipelineRunInputResponse{}, bcode.ErrGetContextBackendData + } + for _, s := range pipelineRun.Status.Steps { + var ( + subStepStatus *v1alpha1.StepStatus + ok bool + ) + if stepName != "" && s.Name != stepName { + subStepStatus, ok = haveSubSteps(s, stepName) + if !ok { + continue + } + subVars := getStepInputs(*subStepStatus, inputsSpec, v, valueFromStep) + stepInputs = append(stepInputs, subVars) + break + } + stepInputs = append(stepInputs, getStepInputs(s.StepStatus, inputsSpec, v, valueFromStep)) + for _, sub := range s.SubStepsStatus { + stepInputs = append(stepInputs, getStepInputs(sub, inputsSpec, v, valueFromStep)) + } + if stepName != "" && s.Name == stepName { + // already found the step + break + } + } + return apis.GetPipelineRunInputResponse{StepInputs: stepInputs}, nil +} + +func haveSubSteps(step v1alpha1.WorkflowStepStatus, subStep string) (*v1alpha1.StepStatus, bool) { + for _, s := range step.SubStepsStatus { + if s.Name == subStep { + return &s, true + } + } + return nil, false +} + +// Copied from stern/stern +var colorList = [][2]*color.Color{ + {color.New(color.FgHiCyan), color.New(color.FgCyan)}, + {color.New(color.FgHiGreen), color.New(color.FgGreen)}, + {color.New(color.FgHiMagenta), color.New(color.FgMagenta)}, + {color.New(color.FgHiYellow), color.New(color.FgYellow)}, + {color.New(color.FgHiBlue), color.New(color.FgBlue)}, + {color.New(color.FgHiRed), color.New(color.FgRed)}, +} + +func determineColor(podName string) (podColor, containerColor *color.Color) { + hash := fnv.New32() + hash.Write([]byte(podName)) + idx := hash.Sum32() % uint32(len(colorList)) + + colors := colorList[idx] + return colors[0], colors[1] +} + +func (p pipelineRunServiceImpl) GetPipelineRunLog(ctx context.Context, pipelineRun apis.PipelineRun, step string) (apis.GetPipelineRunLogResponse, error) { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) + if pipelineRun.Status.ContextBackend == nil { + return apis.GetPipelineRunLogResponse{}, nil + } + + logConfig, err := wfUtils.GetLogConfigFromStep(ctx, p.KubeClient, pipelineRun.Status.ContextBackend.Name, pipelineRun.PipelineName, project.GetNamespace(), step) + if err != nil { + if strings.Contains(err.Error(), "no log config found") { + return apis.GetPipelineRunLogResponse{ + StepBase: getStepBase(pipelineRun, step), + Log: "", + }, nil + } + return apis.GetPipelineRunLogResponse{}, err + } + var logs string + switch { + case logConfig.Data: + logs, err = getResourceLogs(ctx, p.KubeConfig, p.KubeClient, []wfTypes.Resource{{ + Namespace: types2.DefaultKubeVelaNS, + LabelSelector: map[string]string{"app.kubernetes.io/name": "vela-workflow"}, + }}, []string{fmt.Sprintf(`step_name="%s"`, step), fmt.Sprintf("%s/%s", project.GetNamespace(), pipelineRun.PipelineRunName), "cue logs"}) + if err != nil { + return apis.GetPipelineRunLogResponse{}, err + } + case logConfig.Source != nil: + if len(logConfig.Source.Resources) > 0 { + logs, err = getResourceLogs(ctx, p.KubeConfig, p.KubeClient, logConfig.Source.Resources, nil) + if err != nil { + return apis.GetPipelineRunLogResponse{}, err + } + } + if logConfig.Source.URL != "" { + var logsBuilder strings.Builder + readCloser, err := wfUtils.GetLogsFromURL(ctx, logConfig.Source.URL) + if err != nil { + log.Logger.Errorf("get logs from url %s failed: %v", logConfig.Source.URL, err) + return apis.GetPipelineRunLogResponse{}, bcode.ErrReadSourceLog + } + //nolint:errcheck + defer readCloser.Close() + if _, err := io.Copy(&logsBuilder, readCloser); err != nil { + log.Logger.Errorf("copy logs from url %s failed: %v", logConfig.Source.URL, err) + return apis.GetPipelineRunLogResponse{}, bcode.ErrReadSourceLog + } + logs = logsBuilder.String() + } + } + return apis.GetPipelineRunLogResponse{ + StepBase: getStepBase(pipelineRun, step), + Log: logs, + }, nil +} + +func getStepBase(run apis.PipelineRun, step string) apis.StepBase { + for _, s := range run.Status.Steps { + if s.Name == step { + return apis.StepBase{ + ID: s.ID, + Name: s.Name, + Type: s.Type, + Phase: string(s.Phase), + } + } + } + return apis.StepBase{} +} + +func getStepOutputs(step v1alpha1.StepStatus, outputsSpec map[string]v1alpha1.StepOutputs, v *value.Value) apis.StepOutputBase { + o := apis.StepOutputBase{ + StepBase: apis.StepBase{ + Name: step.Name, + ID: step.ID, + Phase: string(step.Phase), + Type: step.Type, + }, + } + values := make([]apis.OutputVar, 0) + for _, output := range outputsSpec[step.Name] { + outputValue, err := v.LookupValue(output.Name) + if err != nil { + continue + } + s, err := outputValue.String() + if err != nil { + continue + } + values = append(values, apis.OutputVar{ + Name: output.Name, + ValueFrom: output.ValueFrom, + Value: s, + }) + } + o.Values = values + return o +} + +func getStepInputs(step v1alpha1.StepStatus, inputsSpec map[string]v1alpha1.StepInputs, v *value.Value, valueFromStep map[string]string) apis.StepInputBase { + o := apis.StepInputBase{ + StepBase: apis.StepBase{ + Name: step.Name, + ID: step.ID, + Phase: string(step.Phase), + Type: step.Type, + }, + } + values := make([]apis.InputVar, 0) + for _, input := range inputsSpec[step.Name] { + outputValue, err := v.LookupValue(input.From) + if err != nil { + continue + } + s, err := outputValue.String() + if err != nil { + continue + } + values = append(values, apis.InputVar{ + Value: s, + From: input.From, + FromStep: valueFromStep[input.From], + ParameterKey: input.ParameterKey, + }) + } + o.Values = values + return o +} + +func getResourceLogs(ctx context.Context, config *rest.Config, cli client.Client, resources []wfTypes.Resource, filters []string) (string, error) { + var ( + linesOfLogKept int64 = 1000 + timeout = 10 * time.Second + errPrint = color.New(color.FgRed, color.Bold).FprintfFunc() + ) + + type PodContainer struct { + Name string + Namespace string + Container string + Label map[string]string + } + pods, err := wfUtils.GetPodListFromResources(ctx, cli, resources) + if err != nil { + log.Logger.Errorf("fail to get pod list from resources: %v", err) + return "", err + } + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + log.Logger.Errorf("fail to get clientset from kubeconfig: %v", err) + return "", err + } + podContainers := make([]PodContainer, 0) + for _, pod := range pods { + for _, container := range pod.Spec.Containers { + pc := PodContainer{ + Name: pod.Name, + Namespace: pod.Namespace, + Container: container.Name, + Label: pod.Labels, + } + podContainers = append(podContainers, pc) + } + } + + wg := sync.WaitGroup{} + logBuilder := strings.Builder{} + logMap := concurrent.NewMap() + wg.Add(len(podContainers)) + + for _, pc := range podContainers { + go func(pc PodContainer) { + defer wg.Done() + ctxQuery, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + podLogOpts := corev1.PodLogOptions{ + Container: pc.Container, + Follow: false, + TailLines: &linesOfLogKept, + } + req := clientSet.CoreV1().Pods(pc.Namespace).GetLogs(pc.Name, &podLogOpts) + podLogs, err := req.Stream(ctxQuery) + if err != nil { + log.Logger.Errorf("fail to get pod logs: %v", err) + return + } + defer func() { + _ = podLogs.Close() + }() + podColor, containerColor := determineColor(pc.Name) + buf := new(bytes.Buffer) + p := podColor.SprintfFunc() + c := containerColor.SprintfFunc() + fmt.Fprintf(buf, "%s %s\n", p("› %s", pc.Name), c("%s", pc.Container)) + var readErr error + r := bufio.NewReader(podLogs) + for { + s, err := r.ReadString('\n') + if err != nil { + if !errors.Is(err, io.EOF) { + readErr = err + } + break + } + if len(filters) != 0 { + for _, f := range filters { + if strings.Contains(s, f) { + buf.WriteString(s) + } + } + } + } + if readErr != nil { + errPrint(buf, "error in copy information from APIServer to buffer: %s", err.Error()) + log.Logger.Errorf("fail to copy pod logs: %v", err) + } + logMap.Store(fmt.Sprintf("%s/%s", pc.Name, pc.Container), buf.String()) + }(pc) + } + wg.Wait() + + order := make([]string, 0) + sort.Strings(order) + logMap.Range(func(key, value any) bool { + order = append(order, key.(string)) + return true + }) + for _, key := range order { + val, ok := logMap.Load(key) + if ok { + logBuilder.WriteString(fmt.Sprintf("%s", val)) + } + } + return logBuilder.String(), nil +} + // RunPipeline will run a pipeline -func (p pipelineServiceImpl) RunPipeline(ctx context.Context, pipeline apis.PipelineBase, req apis.RunPipelineRequest) error { - // todo get context and set to workflowRun +func (p pipelineServiceImpl) RunPipeline(ctx context.Context, pipeline apis.PipelineBase, req apis.RunPipelineRequest) (*apis.PipelineRun, error) { + if err := checkRunMode(&req.Mode); err != nil { + return nil, err + } + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) run := v1alpha1.WorkflowRun{} version := utils.GenerateVersion("") name := fmt.Sprintf("%s-%s", pipeline.Name, version) run.Name = name - run.Namespace = fmt.Sprintf("%s-project", pipeline.Project) - run.Spec.WorkflowRef = pipeline.Name + run.Namespace = project.GetNamespace() + run.Spec.WorkflowSpec = &pipeline.Spec run.Spec.Mode = &req.Mode - return p.KubeClient.Create(ctx, &run) + run.SetLabels(map[string]string{ + labelPipeline: pipeline.Name, + model.LabelSourceOfTruth: model.FromUX, + }) + + // process the context + if req.ContextName != "" { + ppContext, err := p.ContextService.GetContext(ctx, pipeline.Project.Name, pipeline.Name, req.ContextName) + if err != nil { + return nil, err + } + contextData := make(map[string]interface{}) + for _, pair := range ppContext.Values { + contextData[pair.Key] = pair.Value + } + run.Labels[labelContext] = req.ContextName + run.Spec.Context = util.Object2RawExtension(contextData) + } + + if err := p.KubeClient.Create(ctx, &run); err != nil { + return nil, err + } + + return p.PipelineRunService.GetPipelineRun(ctx, apis.PipelineRunMeta{ + PipelineName: pipeline.Name, + Project: apis.NameAlias{Name: project.Name}, + PipelineRunName: name, + }) +} + +// getPipelineInfo returns the pipeline statistic info +// return error can be nil if pipeline hasn't been run +func (p pipelineServiceImpl) getPipelineInfo(ctx context.Context, wf *model.Pipeline, namespace string) (*apis.PipelineInfo, error) { + var runs v1alpha1.WorkflowRunList + err := p.KubeClient.List(context.Background(), &runs, client.InNamespace(namespace), client.MatchingLabels{labelPipeline: wf.Name}) + if err != nil { + if meta.IsNoMatchError(err) { + weekStat := make([]apis.RunStatInfo, 0) + for i := 0; i < 7; i++ { + weekStat = append(weekStat, apis.RunStatInfo{}) + } + return &apis.PipelineInfo{ + LastRun: nil, + RunStat: apis.RunStat{ + ActiveNum: 0, + Total: apis.RunStatInfo{}, + Week: weekStat, + }, + }, nil + } + return nil, err + } + run := getLastRun(runs.Items) + runStat := getRunStat(runs.Items) + pi := &apis.PipelineInfo{ + RunStat: runStat, + } + if run != nil { + projectName := wf.Project + project, err := p.ProjectService.GetProject(ctx, projectName) + if err != nil { + return nil, err + } + pi.LastRun, err = workflowRun2PipelineRun(*run, project, p.ContextService) + if err != nil { + return nil, err + } + } + return pi, nil +} + +func getRunStat(runs []v1alpha1.WorkflowRun) apis.RunStat { + today := time.Now().Unix() + isActive := func(run v1alpha1.WorkflowRun) bool { + return !run.Status.Finished && !run.Status.Terminated && run.Status.Suspend + } + isSuccess := func(run v1alpha1.WorkflowRun) bool { + return run.Status.Phase == v1alpha1.WorkflowStateSucceeded + } + // returned int x means the (x+1)/7 days of the week, valid number is 0-6 + inThisWeek := func(run v1alpha1.WorkflowRun) (int, bool) { + startTime := run.Status.StartTime.Time.Unix() + // one week, note this is not week of natual, but week of unix timestamp + if today-startTime < 604800 { + return 6 - int((today-startTime)/86400), true + } + return -1, false + } + var ( + act int + success int + fail int + week = make([]apis.RunStatInfo, 7) + ) + + for _, run := range runs { + // total = success + fail + active + day, inWeek := inThisWeek(run) + if inWeek { + week[day].Total++ + } + if isActive(run) { + act++ + } else { + if isSuccess(run) { + if inWeek { + week[day].Success++ + } + success++ + } else { + fail++ + if inWeek { + week[day].Fail++ + } + } + } + + } + return apis.RunStat{ + ActiveNum: act, + Total: apis.RunStatInfo{ + Total: len(runs), + Success: success, + Fail: fail, + }, + Week: week, + } +} + +func getLastRun(runs []v1alpha1.WorkflowRun) *v1alpha1.WorkflowRun { + if len(runs) == 0 { + return nil + } + last := runs[0] + lastStartTime := last.Status.StartTime.Time + for _, wfr := range runs { + wfr.Status.StartTime.After(lastStartTime) + last = wfr + lastStartTime = wfr.Status.StartTime.Time + } + // We don't need managed fields, save some bandwidth + last.ManagedFields = nil + return &last } // GetPipelineRun will get a pipeline run -func (p pipelineRunServiceImpl) GetPipelineRun(ctx context.Context, meta apis.PipelineRunMeta) (apis.PipelineRun, error) { - namespacedName := client.ObjectKey{Name: meta.PipelineName, Namespace: nsForProj(meta.Project)} +func (p pipelineRunServiceImpl) GetPipelineRun(ctx context.Context, meta apis.PipelineRunMeta) (*apis.PipelineRun, error) { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) + namespacedName := client.ObjectKey{Name: meta.PipelineRunName, Namespace: project.GetNamespace()} run := v1alpha1.WorkflowRun{} if err := p.KubeClient.Get(ctx, namespacedName, &run); err != nil { - return apis.PipelineRun{}, err + return nil, err } - return workflowRun2PipelineRun(run), nil + if run.Labels != nil && run.Labels[labelPipeline] != "" { + pipeline := &model.Pipeline{ + Name: run.Labels[labelPipeline], + Project: project.Name, + } + if err := p.Store.Get(ctx, pipeline); err != nil { + log.Logger.Errorf("failed to load the workflow %s", err.Error()) + if errors.Is(err, datastore.ErrRecordNotExist) { + return nil, bcode.ErrPipelineNotExist + } + return nil, err + } + run.Spec.WorkflowSpec = &pipeline.Spec + + } + return workflowRun2PipelineRun(run, project, p.ContextService) } // ListPipelineRuns will list all pipeline runs func (p pipelineRunServiceImpl) ListPipelineRuns(ctx context.Context, base apis.PipelineBase) (apis.ListPipelineRunResponse, error) { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) wfrs := v1alpha1.WorkflowRunList{} - if err := p.KubeClient.List(ctx, &wfrs, client.InNamespace(nsForProj(base.Project))); err != nil { + if err := p.KubeClient.List(ctx, &wfrs, client.InNamespace(project.GetNamespace()), client.MatchingLabels{labelPipeline: base.Name}); err != nil { return apis.ListPipelineRunResponse{}, err } - res := apis.ListPipelineRunResponse{} - for _, wfr := range wfrs.Items { - if wfr.Spec.WorkflowRef == base.Name { - res.Runs = append(res.Runs, p.workflowRun2runBriefing(ctx, wfr)) - } + res := apis.ListPipelineRunResponse{ + Runs: make([]apis.PipelineRunBriefing, 0), } + for _, wfr := range wfrs.Items { + res.Runs = append(res.Runs, p.workflowRun2runBriefing(ctx, wfr, project)) + } + res.Total = int64(len(res.Runs)) return res, nil } // DeletePipelineRun will delete a pipeline run func (p pipelineRunServiceImpl) DeletePipelineRun(ctx context.Context, meta apis.PipelineRunMeta) error { - namespacedName := client.ObjectKey{Name: meta.PipelineName, Namespace: nsForProj(meta.Project)} - run := v1alpha1.WorkflowRun{} - if err := p.KubeClient.Get(ctx, namespacedName, &run); err != nil { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) + run := v1alpha1.WorkflowRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: meta.PipelineRunName, + Namespace: project.GetNamespace(), + }, + } + err := p.KubeClient.Delete(ctx, &run) + return client.IgnoreNotFound(err) +} + +// CleanPipelineRuns will clean all pipeline runs, it equals to call ListPipelineRuns and multiple DeletePipelineRun +func (p pipelineRunServiceImpl) CleanPipelineRuns(ctx context.Context, base apis.PipelineBase) error { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) + wfrs := v1alpha1.WorkflowRunList{} + if err := p.KubeClient.List(ctx, &wfrs, client.InNamespace(project.GetNamespace()), client.MatchingLabels{labelPipeline: base.Name}); err != nil { return err } - return p.KubeClient.Delete(ctx, &run) + for _, wfr := range wfrs.Items { + if err := p.KubeClient.Delete(ctx, wfr.DeepCopy()); err != nil { + return client.IgnoreNotFound(err) + } + } + return nil +} + +// InitContext will init pipeline context record +func (c contextServiceImpl) InitContext(ctx context.Context, projectName, pipelineName string) (*model.PipelineContext, error) { + modelCtx := model.PipelineContext{ + ProjectName: projectName, + PipelineName: pipelineName, + } + if err := c.Store.Get(ctx, &modelCtx); err == nil { + return nil, bcode.ErrContextAlreadyExist + } + modelCtx.Contexts = make(map[string][]model.Value) + if err := c.Store.Add(ctx, &modelCtx); err != nil { + return nil, err + } + return &modelCtx, nil } // GetContext will get a context @@ -263,29 +962,36 @@ func (c contextServiceImpl) GetContext(ctx context.Context, projectName, pipelin } vals, ok := modelCtx.Contexts[name] if !ok { - return nil, errors.New("context not found") + return nil, bcode.ErrContextNotFound } return &apis.Context{Name: name, Values: vals}, nil } // CreateContext will create a context func (c contextServiceImpl) CreateContext(ctx context.Context, projectName, pipelineName string, context apis.Context) (*model.PipelineContext, error) { - modelCtx := model.PipelineContext{ + modelCtx := &model.PipelineContext{ ProjectName: projectName, PipelineName: pipelineName, } - if err := c.Store.Get(ctx, &modelCtx); err != nil { - return nil, err + if err := c.Store.Get(ctx, modelCtx); err != nil { + if errors.Is(err, datastore.ErrRecordNotExist) { + modelCtx, err = c.InitContext(ctx, projectName, pipelineName) + if err != nil { + return nil, err + } + } else { + return nil, err + } } if _, ok := modelCtx.Contexts[context.Name]; ok { log.Logger.Errorf("context %s already exists", context.Name) return nil, bcode.ErrContextAlreadyExist } modelCtx.Contexts[context.Name] = context.Values - if err := c.Store.Put(ctx, &modelCtx); err != nil { + if err := c.Store.Put(ctx, modelCtx); err != nil { return nil, err } - return &modelCtx, nil + return modelCtx, nil } // UpdateContext will update a context @@ -311,6 +1017,12 @@ func (c contextServiceImpl) ListContexts(ctx context.Context, projectName, pipel PipelineName: pipelineName, } if err := c.Store.Get(ctx, &modelCtx); err != nil { + if errors.Is(err, datastore.ErrRecordNotExist) { + return &apis.ListContextValueResponse{ + Total: 0, + Contexts: make(map[string][]model.Value), + }, nil + } return nil, err } return &apis.ListContextValueResponse{ @@ -329,99 +1041,164 @@ func (c contextServiceImpl) DeleteContext(ctx context.Context, projectName, pipe return err } delete(modelCtx.Contexts, name) - if err := c.Store.Put(ctx, &modelCtx); err != nil { - return err + return c.Store.Put(ctx, &modelCtx) +} + +// DeleteAllContexts will delete all contexts of a pipeline +func (c contextServiceImpl) DeleteAllContexts(ctx context.Context, projectName, pipelineName string) error { + modelCtx := model.PipelineContext{ + ProjectName: projectName, + PipelineName: pipelineName, } - return nil + return c.Store.Delete(ctx, &modelCtx) } -func nsForProj(proj string) string { - return fmt.Sprintf("project-%s", proj) -} - -func getWfDescription(wf v1alpha1.Workflow) string { - if wf.Labels == nil { - return "" - } - return wf.Labels[labelDescription] -} - -func getWfAlias(wf v1alpha1.Workflow) string { - if wf.Labels == nil { - return "" - } - return wf.Labels[labelAlias] -} - -func fuzzyMatch(wf v1alpha1.Workflow, q string) bool { +func fuzzyMatch(wf *model.Pipeline, q string) bool { if strings.Contains(wf.Name, q) { return true } - if strings.Contains(getWfAlias(wf), q) { + if strings.Contains(wf.Alias, q) { return true } - if strings.Contains(getWfDescription(wf), q) { + if strings.Contains(wf.Description, q) { return true } return false } -func workflow2PipelineBase(wf v1alpha1.Workflow) *apis.PipelineBase { - project := strings.TrimRight(wf.Namespace, "-project") +func pipeline2PipelineBase(wf *model.Pipeline, project model.Project) *apis.PipelineBase { return &apis.PipelineBase{ PipelineMeta: apis.PipelineMeta{ - Name: wf.Name, - Project: project, - Description: getWfDescription(wf), - Alias: getWfAlias(wf), + Name: wf.Name, + Project: apis.NameAlias{ + Name: project.Name, + Alias: project.Alias, + }, + Description: wf.Description, + Alias: wf.Alias, + CreateTime: wf.CreateTime, }, - Spec: wf.WorkflowSpec, + Spec: wf.Spec, } } -func workflowRun2PipelineRun(run v1alpha1.WorkflowRun) apis.PipelineRun { - return apis.PipelineRun{ +func workflowRun2PipelineRun(run v1alpha1.WorkflowRun, project *model.Project, ctxService ContextService) (*apis.PipelineRun, error) { + mergeSteps(&run) + pipelineName := run.Labels[labelPipeline] + pipelineRun := &apis.PipelineRun{ PipelineRunBase: apis.PipelineRunBase{ PipelineRunMeta: apis.PipelineRunMeta{ - PipelineName: run.Spec.WorkflowRef, - Project: strings.TrimRight(run.Namespace, "-project"), + PipelineName: pipelineName, + Project: apis.NameAlias{ + Name: project.Name, + Alias: project.Alias, + }, PipelineRunName: run.Name, }, + Spec: run.Spec, }, Status: run.Status, } + if labels := run.GetLabels(); labels != nil { + ctxName := labels[labelContext] + if ctxName != "" { + ctx, err := ctxService.GetContext(context.Background(), project.Name, pipelineName, ctxName) + if err != nil && !errors.Is(err, datastore.ErrRecordNotExist) { + return nil, err + } + pipelineRun.PipelineRunBase.ContextName = ctxName + pipelineRun.PipelineRunBase.ContextValues = ctx.Values + } + } + return pipelineRun, nil } -func (p pipelineRunServiceImpl) workflowRun2runBriefing(ctx context.Context, run v1alpha1.WorkflowRun) apis.PipelineRunBriefing { - contextName := run.Labels[labelContextName] - project := strings.TrimRight(run.Namespace, "-project") - apiContext, err := p.ContextService.GetContext(ctx, project, run.Spec.WorkflowRef, contextName) - if err != nil { - log.Logger.Warnf("failed to get pipeline run context %s/%s/%s: %v", project, run.Spec.WorkflowRef, contextName, err) - apiContext = nil +func mergeSteps(run *v1alpha1.WorkflowRun) { + if run.Spec.WorkflowSpec == nil { + return + } + if run.Status.Steps == nil { + run.Status.Steps = make([]v1alpha1.WorkflowStepStatus, 0) + } + var stepStatus = make(map[string]*v1alpha1.WorkflowStepStatus, len(run.Status.Steps)) + for i, step := range run.Status.Steps { + stepStatus[step.Name] = &run.Status.Steps[i] + } + for _, step := range run.Spec.WorkflowSpec.Steps { + if stepStatusCache, exist := stepStatus[step.Name]; !exist { + var subSteps []v1alpha1.StepStatus + for _, subStep := range step.SubSteps { + subSteps = append(subSteps, v1alpha1.StepStatus{ + Name: subStep.Name, + Type: subStep.Type, + Phase: v1alpha1.WorkflowStepPhasePending, + }) + } + run.Status.Steps = append(run.Status.Steps, v1alpha1.WorkflowStepStatus{ + StepStatus: v1alpha1.StepStatus{ + Name: step.Name, + Type: step.Type, + Phase: v1alpha1.WorkflowStepPhasePending, + }, + SubStepsStatus: subSteps, + }) + } else if len(step.SubSteps) > len(stepStatusCache.SubStepsStatus) { + var subStepStatus = make(map[string]v1alpha1.StepStatus, len(stepStatusCache.SubStepsStatus)) + for i, step := range stepStatusCache.SubStepsStatus { + subStepStatus[step.Name] = stepStatusCache.SubStepsStatus[i] + } + for _, subStep := range step.SubSteps { + if _, exist := subStepStatus[subStep.Name]; !exist { + stepStatusCache.SubStepsStatus = append(stepStatusCache.SubStepsStatus, v1alpha1.StepStatus{ + Name: subStep.Name, + Type: subStep.Type, + Phase: v1alpha1.WorkflowStepPhasePending, + }) + } + } + } + } +} + +func (p pipelineRunServiceImpl) workflowRun2runBriefing(ctx context.Context, run v1alpha1.WorkflowRun, project *model.Project) apis.PipelineRunBriefing { + var ( + apiContext *apis.Context + err error + ) + pipelineName := run.Labels[labelPipeline] + if contextName, ok := run.Labels[labelContext]; ok { + apiContext, err = p.ContextService.GetContext(ctx, project.Name, pipelineName, contextName) + if err != nil { + log.Logger.Warnf("failed to get pipeline run context %s/%s/%s: %v", project, pipelineName, contextName, err) + apiContext = nil + } } - return apis.PipelineRunBriefing{ + briefing := apis.PipelineRunBriefing{ PipelineRunName: run.Name, Finished: run.Status.Finished, Phase: run.Status.Phase, Message: run.Status.Message, StartTime: run.Status.StartTime, EndTime: run.Status.EndTime, - ContextName: apiContext.Name, - ContextValues: apiContext.Values, } + if apiContext != nil { + briefing.ContextName = apiContext.Name + briefing.ContextValues = apiContext.Values + } + return briefing } -func (p pipelineRunServiceImpl) checkRecordRunning(ctx context.Context, pipelineRun apis.PipelineRunBase) (*v1alpha1.WorkflowRun, error) { +func (p pipelineRunServiceImpl) checkRunNotFinished(ctx context.Context, pipelineRun apis.PipelineRunBase) (*v1alpha1.WorkflowRun, error) { + project := ctx.Value(&apis.CtxKeyProject).(*model.Project) run := v1alpha1.WorkflowRun{} if err := p.KubeClient.Get(ctx, types.NamespacedName{ - Namespace: nsForProj(pipelineRun.Project), + Namespace: project.GetNamespace(), Name: pipelineRun.PipelineRunName, }, &run); err != nil { return nil, err } - if !run.Status.Suspend && !run.Status.Terminated && !run.Status.Finished { - return nil, fmt.Errorf("workflow is still running, can not operate a running workflow") + if run.Status.Terminated || run.Status.Finished { + return nil, bcode.ErrPipelineRunFinished } return &run, nil } @@ -461,3 +1238,59 @@ func (p pipelineRunServiceImpl) terminatePipelineRun(ctx context.Context, run *v return nil } + +func checkPipelineSpec(spec v1alpha1.WorkflowSpec) error { + if len(spec.Steps) == 0 { + return bcode.ErrNoSteps + } + return nil +} + +func checkRunMode(mode *v1alpha1.WorkflowExecuteMode) error { + if mode.Steps == "" { + mode.Steps = v1alpha1.WorkflowModeStep + } + if mode.SubSteps == "" { + mode.SubSteps = v1alpha1.WorkflowModeDAG + } + if mode.Steps != v1alpha1.WorkflowModeStep && mode.Steps != v1alpha1.WorkflowModeDAG && + mode.SubSteps != v1alpha1.WorkflowModeStep && mode.SubSteps != v1alpha1.WorkflowModeDAG { + return bcode.ErrWrongMode + } + return nil +} + +// NewTestPipelineService create the pipeline service instance for testing +func NewTestPipelineService(ds datastore.DataStore, c client.Client, cfg *rest.Config) PipelineService { + projectService := NewTestProjectService(ds, c) + contextService := NewTestContextService(ds) + ppRunService := NewTestPipelineRunService(ds, c, cfg) + pipelineService := &pipelineServiceImpl{ + ProjectService: projectService, + ContextService: contextService, + KubeClient: c, + KubeConfig: cfg, + PipelineRunService: ppRunService, + Store: ds, + } + return pipelineService +} + +// NewTestPipelineRunService create the pipeline run service instance for testing +func NewTestPipelineRunService(ds datastore.DataStore, c client.Client, cfg *rest.Config) PipelineRunService { + contextService := NewTestContextService(ds) + projectService := NewTestProjectService(ds, c) + return &pipelineRunServiceImpl{ + KubeClient: c, + KubeConfig: cfg, + ContextService: contextService, + ProjectService: projectService, + } +} + +// NewTestContextService create the context service instance for testing +func NewTestContextService(ds datastore.DataStore) ContextService { + return &contextServiceImpl{ + Store: ds, + } +} diff --git a/pkg/apiserver/domain/service/pipeline_test.go b/pkg/apiserver/domain/service/pipeline_test.go new file mode 100644 index 000000000..5214a6254 --- /dev/null +++ b/pkg/apiserver/domain/service/pipeline_test.go @@ -0,0 +1,143 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + + "github.com/kubevela/workflow/api/v1alpha1" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/oam-dev/kubevela/pkg/apiserver/domain/model" + "github.com/oam-dev/kubevela/pkg/apiserver/infrastructure/datastore" + apisv1 "github.com/oam-dev/kubevela/pkg/apiserver/interfaces/api/dto/v1" + "github.com/oam-dev/kubevela/pkg/oam/util" +) + +var ( + // defaultNamespace = "project-default-ns1-test" + pipelineService *pipelineServiceImpl + pipelineRunService *pipelineRunServiceImpl + userService *userServiceImpl + contextService *contextServiceImpl + projectService *projectServiceImpl + ctx context.Context + + pipelineName = "test-pipeline" + projectName = "test-project" +) +var _ = Describe("Test pipeline service functions", func() { + It("Init services and project", func() { + ds, err := NewDatastore(datastore.Config{Type: "kubeapi", Database: "pipeline-test-kubevela"}) + Expect(ds).ToNot(BeNil()) + Expect(err).Should(BeNil()) + Expect(err).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + pipelineService = NewTestPipelineService(ds, k8sClient, cfg).(*pipelineServiceImpl) + pipelineRunService = pipelineService.PipelineRunService.(*pipelineRunServiceImpl) + contextService = pipelineService.ContextService.(*contextServiceImpl) + projectService = pipelineService.ProjectService.(*projectServiceImpl) + userService = &userServiceImpl{Store: ds, K8sClient: k8sClient} + + ctx = context.WithValue(context.TODO(), &apisv1.CtxKeyUser, "admin") + err = userService.Init(context.TODO()) + Expect(err).Should(BeNil()) + _, err = projectService.CreateProject(ctx, apisv1.CreateProjectRequest{ + Name: projectName, + Owner: "admin", + }) + Expect(err).Should(BeNil()) + projModel, err := projectService.GetProject(context.TODO(), projectName) + Expect(err).Should(BeNil()) + ctx = context.WithValue(ctx, &apisv1.CtxKeyProject, projModel) + }) + + It("Test create pipeline", func() { + rawProps := []byte(`{"url":"https://api.github.com/repos/kubevela/kubevela"}`) + testPipelineSteps := []v1alpha1.WorkflowStep{ + { + SubSteps: []v1alpha1.WorkflowStepBase{ + { + Name: "request", + Type: "request", + Outputs: v1alpha1.StepOutputs{ + { + ValueFrom: "import \"strconv\"\n\"Current star count: \" + strconv.FormatInt(response[\"stargazers_count\"], 10)\n", + Name: "stars", + }, + }, + Properties: &runtime.RawExtension{ + Raw: rawProps, + }, + }, + }, + WorkflowStepBase: v1alpha1.WorkflowStepBase{ + Name: "step-group", + Type: "step-group", + }, + }, + } + + By("create pipeline with sub-steps") + pipeline, err := pipelineService.CreatePipeline(ctx, apisv1.CreatePipelineRequest{ + Name: pipelineName, + Spec: v1alpha1.WorkflowSpec{ + Steps: testPipelineSteps, + }, + }) + Expect(err).Should(BeNil()) + Expect(pipeline.Name).Should(Equal(pipelineName)) + Expect(pipeline.Spec.Steps[0].Name).Should(Equal("step-group")) + }) + + It("list pipeline", func() { + pipelines, err := pipelineService.ListPipelines(ctx, apisv1.ListPipelineRequest{ + Detailed: true, + }) + Expect(err).Should(BeNil()) + Expect(pipelines).ShouldNot(BeNil()) + Expect(pipelines.Total).Should(Equal(1)) + Expect(len(pipelines.Pipelines)).Should(Equal(1)) + Expect(pipelines.Pipelines[0].Info).ShouldNot(BeNil()) + }) + + It("get pipeline contexts", func() { + By("no context") + contexts, err := contextService.ListContexts(ctx, projectName, pipelineName) + Expect(err).Should(BeNil()) + Expect(contexts.Total).Should(Equal(0)) + Expect(len(contexts.Contexts)).Should(Equal(0)) + + By("create context") + contextName := "test-context" + contextKey := "test-key" + contextVal := "test-val" + ppCtx := apisv1.Context{ + Name: contextName, + Values: []model.Value{ + { + Key: contextKey, + Value: contextVal, + }, + }, + } + context, err := contextService.CreateContext(ctx, projectName, pipelineName, ppCtx) + Expect(err).Should(BeNil()) + Expect(len(context.Contexts)).Should(Equal(1)) + }) +}) diff --git a/pkg/apiserver/domain/service/project.go b/pkg/apiserver/domain/service/project.go index 73227e896..94fe75db8 100644 --- a/pkg/apiserver/domain/service/project.go +++ b/pkg/apiserver/domain/service/project.go @@ -22,6 +22,7 @@ import ( "fmt" terraformapi "github.com/oam-dev/terraform-controller/api/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/oam-dev/kubevela/apis/types" @@ -31,6 +32,7 @@ import ( "github.com/oam-dev/kubevela/pkg/apiserver/utils/bcode" "github.com/oam-dev/kubevela/pkg/apiserver/utils/log" "github.com/oam-dev/kubevela/pkg/multicluster" + "github.com/oam-dev/kubevela/pkg/utils" ) // ProjectService project manage service. @@ -159,6 +161,13 @@ func (p *projectServiceImpl) GetProject(ctx context.Context, projectName string) } return nil, err } + if _, err := utils.GetNamespace(ctx, p.K8sClient, project.GetNamespace()); err != nil { + if apierrors.IsNotFound(err) { + if err := utils.CreateNamespace(ctx, p.K8sClient, projectName); err != nil && !apierrors.IsAlreadyExists(err) { + return nil, bcode.ErrProjectNamespaceFail + } + } + } return project, nil } @@ -319,11 +328,16 @@ func (p *projectServiceImpl) CreateProject(ctx context.Context, req apisv1.Creat } } + if err := utils.CreateNamespace(ctx, p.K8sClient, req.Name); err != nil && !apierrors.IsAlreadyExists(err) { + return nil, bcode.ErrProjectNamespaceFail + } + newProject := &model.Project{ Name: req.Name, Description: req.Description, Alias: req.Alias, Owner: owner, + Namespace: req.Name, } if err := p.Store.Add(ctx, newProject); err != nil { @@ -526,6 +540,7 @@ func ConvertProjectModel2Base(project *model.Project, owner *model.User) *apisv1 CreateTime: project.CreateTime, UpdateTime: project.UpdateTime, Owner: apisv1.NameAlias{Name: project.Owner}, + Namespace: project.GetNamespace(), } if owner != nil && owner.Name == project.Owner { base.Owner = apisv1.NameAlias{Name: owner.Name, Alias: owner.Alias} diff --git a/pkg/apiserver/domain/service/rbac.go b/pkg/apiserver/domain/service/rbac.go index eb626f3e3..22c909773 100644 --- a/pkg/apiserver/domain/service/rbac.go +++ b/pkg/apiserver/domain/service/rbac.go @@ -59,6 +59,7 @@ var defaultProjectPermissionTemplate = []*model.PermissionTemplate{ "project:{projectName}/permission:*", "project:{projectName}/environment:*", "project:{projectName}/application:*/*", + "project:{projectName}/pipeline:*/*", }, Actions: []string{"detail", "list"}, Effect: "Allow", @@ -96,6 +97,16 @@ var defaultProjectPermissionTemplate = []*model.PermissionTemplate{ Effect: "Allow", Scope: "project", }, + { + Name: "pipeline-management", + Alias: "Pipeline Management", + Resources: []string{ + "project:{projectName}/pipeline:*", + }, + Actions: []string{"*"}, + Effect: "Allow", + Scope: "project", + }, } var defaultPlatformPermission = []*model.PermissionTemplate{ @@ -234,6 +245,17 @@ var ResourceMaps = map[string]resourceMetadata{ pathName: "configName", }, "provider": {}, + "pipeline": { + pathName: "pipelineName", + subResources: map[string]resourceMetadata{ + "context": { + pathName: "contextName", + }, + "pipelineRun": { + pathName: "pipelineRunName", + }, + }, + }, }, pathName: "projectName", }, @@ -866,7 +888,7 @@ func (p *rbacServiceImpl) InitDefaultRoleAndUsersForProject(ctx context.Context, }, &model.Role{ Name: "project-admin", Alias: "Project Admin", - Permissions: []string{"project-view", "app-management", "env-management", "role-management", "configuration-read"}, + Permissions: []string{"project-view", "app-management", "env-management", "role-management", "pipeline-management", "configuration-read"}, Project: project.Name, }, &model.Role{ Name: "project-viewer", diff --git a/pkg/apiserver/domain/service/rbac_test.go b/pkg/apiserver/domain/service/rbac_test.go index 11c442c51..3cc73d0ca 100644 --- a/pkg/apiserver/domain/service/rbac_test.go +++ b/pkg/apiserver/domain/service/rbac_test.go @@ -198,7 +198,7 @@ var _ = Describe("Test rbac service", func() { policies, err := rbacService.ListPermissions(context.TODO(), "init-test") Expect(err).Should(BeNil()) - Expect(len(policies)).Should(BeEquivalentTo(int64(5))) + Expect(len(policies)).Should(BeEquivalentTo(int64(6))) }) It("Test UpdatePermission", func() { diff --git a/pkg/apiserver/domain/service/suite_test.go b/pkg/apiserver/domain/service/suite_test.go index 850a910b8..39bcb808b 100644 --- a/pkg/apiserver/domain/service/suite_test.go +++ b/pkg/apiserver/domain/service/suite_test.go @@ -25,6 +25,7 @@ import ( "testing" "time" + "github.com/kubevela/workflow/api/v1alpha1" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -74,7 +75,10 @@ var _ = BeforeSuite(func(done Done) { By("new kube client") cfg.Timeout = time.Minute * 2 - k8sClient, err = client.New(cfg, client.Options{Scheme: common.Scheme}) + scheme := common.Scheme + err = v1alpha1.AddToScheme(scheme) + Expect(err).ShouldNot(HaveOccurred()) + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) Expect(err).Should(BeNil()) Expect(k8sClient).ToNot(BeNil()) By("new kube client success") diff --git a/pkg/apiserver/domain/service/workflow_test.go b/pkg/apiserver/domain/service/workflow_test.go index 7b3f5f265..cee26d258 100644 --- a/pkg/apiserver/domain/service/workflow_test.go +++ b/pkg/apiserver/domain/service/workflow_test.go @@ -58,7 +58,7 @@ var _ = Describe("Test workflow service functions", func() { Expect(ds).ToNot(BeNil()) Expect(err).Should(BeNil()) rbacService := &rbacServiceImpl{Store: ds} - projectService = &projectServiceImpl{Store: ds, RbacService: rbacService} + projectService = &projectServiceImpl{Store: ds, RbacService: rbacService, K8sClient: k8sClient} envService = &envServiceImpl{Store: ds, KubeClient: k8sClient, ProjectService: projectService} envBinding = &envBindingServiceImpl{ Store: ds, diff --git a/pkg/apiserver/infrastructure/clients/kubeclient.go b/pkg/apiserver/infrastructure/clients/kubeclient.go index 436773a31..a92e03437 100644 --- a/pkg/apiserver/infrastructure/clients/kubeclient.go +++ b/pkg/apiserver/infrastructure/clients/kubeclient.go @@ -20,13 +20,13 @@ import ( "fmt" pkgmulticluster "github.com/kubevela/pkg/multicluster" + "github.com/kubevela/workflow/api/v1alpha1" + "github.com/kubevela/workflow/pkg/cue/packages" "k8s.io/client-go/discovery" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" - "github.com/kubevela/workflow/pkg/cue/packages" - apiConfig "github.com/oam-dev/kubevela/pkg/apiserver/config" "github.com/oam-dev/kubevela/pkg/auth" "github.com/oam-dev/kubevela/pkg/oam/discoverymapper" @@ -73,11 +73,13 @@ func GetKubeClient() (client.Client, error) { if kubeConfig == nil { return nil, fmt.Errorf("please call SetKubeConfig first") } - var err error - kubeClient, err = pkgmulticluster.NewClient(kubeConfig, pkgmulticluster.ClientOptions{ + err := v1alpha1.AddToScheme(common.Scheme) + if err != nil { + return nil, err + } + return pkgmulticluster.NewClient(kubeConfig, pkgmulticluster.ClientOptions{ Options: client.Options{Scheme: common.Scheme}, }) - return kubeClient, err } // GetKubeConfig create/get kube runtime config diff --git a/pkg/apiserver/interfaces/api/dto/v1/types.go b/pkg/apiserver/interfaces/api/dto/v1/types.go index a8a8ad298..0138b4c6d 100644 --- a/pkg/apiserver/interfaces/api/dto/v1/types.go +++ b/pkg/apiserver/interfaces/api/dto/v1/types.go @@ -50,12 +50,14 @@ var ( CtxKeyApplicationComponent = "component" // CtxKeyUser request context key of user CtxKeyUser = "user" + // CtxKeyProject request context key of project + CtxKeyProject = "project" // CtxKeyToken request context key of request token CtxKeyToken = "token" // CtxKeyPipeline request context key of pipeline CtxKeyPipeline = "pipeline" - // CtxKeyPipelineContex request context key of pipeline context - CtxKeyPipelineContex = "pipeline-context" + // CtxKeyPipelineContext request context key of pipeline context + CtxKeyPipelineContext = "pipeline-context" // CtxKeyPipelineRun request context key of pipeline run CtxKeyPipelineRun = "pipeline-run" ) @@ -791,6 +793,7 @@ type ProjectBase struct { CreateTime time.Time `json:"createTime"` UpdateTime time.Time `json:"updateTime"` Owner NameAlias `json:"owner,omitempty"` + Namespace string `json:"-"` } // CreateProjectRequest create project request body @@ -1551,10 +1554,11 @@ type ListConfigDistributionResponse struct { // PipelineMeta is metadata of pipeline type PipelineMeta struct { - Name string `json:"name" validate:"checkname"` - Alias string `json:"alias" validate:"checkalias" optional:"true"` - Project string `json:"project"` - Description string `json:"description" optional:"true"` + Name string `json:"name"` + Alias string `json:"alias"` + Project NameAlias `json:"project"` + Description string `json:"description"` + CreateTime time.Time `json:"createTime"` } // PipelineBase is the base info of pipeline @@ -1580,7 +1584,6 @@ type RunStat struct { // CreatePipelineRequest is the request body of creating pipeline type CreatePipelineRequest struct { Name string `json:"name" validate:"checkname"` - Project string `json:"project"` Alias string `json:"alias" validate:"checkalias" optional:"true"` Description string `json:"description" optional:"true"` Spec workflowv1alpha1.WorkflowSpec `json:"spec"` @@ -1593,8 +1596,9 @@ type PipelineMetaResponse struct { // ListPipelineRequest is the request body of listing pipeline type ListPipelineRequest struct { - Projects []string `json:"projects"` - Query string `json:"query"` + Projects []string `json:"projects" optional:"true"` + Query string `json:"query" optional:"true"` + Detailed bool `json:"detailed" optional:"true"` } // ListPipelineResponse is the response body of listing pipeline @@ -1616,11 +1620,6 @@ type UpdatePipelineRequest struct { Spec workflowv1alpha1.WorkflowSpec `json:"spec" optional:"true"` } -// GetPipelineRequest is the request body of getting pipeline -type GetPipelineRequest struct { - Detailed bool `json:"detailed"` -} - // GetPipelineResponse is the response body of getting pipeline type GetPipelineResponse struct { PipelineBase `json:",inline"` @@ -1629,9 +1628,8 @@ type GetPipelineResponse struct { // PipelineInfo is the info of pipeline type PipelineInfo struct { - RelatedApps []ApplicationBase `json:"relatedApps"` - LastRunStatus workflowv1alpha1.WorkflowRunStatus `json:"lastRunStatus"` - RunStat RunStat `json:"runStat"` + LastRun *PipelineRun `json:"lastRun"` + RunStat RunStat `json:"runStat"` } /***********************/ @@ -1652,9 +1650,9 @@ type PipelineRunBriefing struct { // PipelineRunMeta is the metadata of pipeline run type PipelineRunMeta struct { - PipelineName string `json:"pipelineName"` - Project string `json:"project"` - PipelineRunName string `json:"pipelineRunName"` + PipelineName string `json:"pipelineName"` + Project NameAlias `json:"project"` + PipelineRunName string `json:"pipelineRunName"` } // PipelineRun is the info of pipeline run @@ -1667,14 +1665,16 @@ type PipelineRun struct { type PipelineRunBase struct { PipelineRunMeta `json:",inline"` // Record marks the run of the pipeline - Record int64 `json:"record"` - ContextName string `json:"contextName"` - Spec workflowv1alpha1.WorkflowRunSpec `json:"spec"` + Record int64 `json:"record"` + ContextName string `json:"contextName"` + ContextValues []model.Value `json:"contextValues"` + Spec workflowv1alpha1.WorkflowRunSpec `json:"spec"` } // RunPipelineRequest is the request body of running pipeline type RunPipelineRequest struct { // Mode is the mode of the pipeline run. Available values are: "StepByStep", "DAG" for both `step` and `subStep` + // default: "StepByStep" for `step`, "DAG" for `subStep` Mode workflowv1alpha1.WorkflowExecuteMode `json:"mode" optional:"true"` ContextName string `json:"contextName"` } @@ -1687,12 +1687,18 @@ type ListPipelineRunResponse struct { // GetPipelineRunLogResponse is the response body of getting pipeline run log type GetPipelineRunLogResponse struct { - Log []Log `json:"log"` + StepBase `json:",inline"` + Log string `json:"log"` } // GetPipelineRunOutputResponse is the response body of getting pipeline run output type GetPipelineRunOutputResponse struct { - Output []Output `json:"output"` + StepOutputs []StepOutputBase `json:"outputs"` +} + +// GetPipelineRunInputResponse is the response body of getting pipeline run input +type GetPipelineRunInputResponse struct { + StepInputs []StepInputBase `json:"inputs"` } // StepBase is the base info of step @@ -1703,16 +1709,31 @@ type StepBase struct { Phase string `json:"phase"` } -// Log is the log of step -type Log struct { +// StepOutputBase is the output of step +type StepOutputBase struct { StepBase `json:",inline"` - Log string `json:"log"` + Values []OutputVar `json:"values"` } -// Output is the output of step -type Output struct { +// StepInputBase is the input of step +type StepInputBase struct { StepBase `json:",inline"` - Vars map[string]string `json:"vars"` + Values []InputVar `json:"values"` +} + +// OutputVar is one output var +type OutputVar struct { + Name string `json:"name"` + Value string `json:"value"` + ValueFrom string `json:"valueFrom"` +} + +// InputVar is one input var +type InputVar struct { + From string `json:"from"` + FromStep string `json:"fromStep"` + ParameterKey string `json:"parameterKey"` + Value string `json:"value"` } /*******************/ diff --git a/pkg/apiserver/interfaces/api/interfaces.go b/pkg/apiserver/interfaces/api/interfaces.go index e44cc302c..c66f07adc 100644 --- a/pkg/apiserver/interfaces/api/interfaces.go +++ b/pkg/apiserver/interfaces/api/interfaces.go @@ -62,6 +62,7 @@ func InitAPIBean() []interface{} { RegisterAPIInterface(NewApplicationAPIInterface()) RegisterAPIInterface(NewProjectAPIInterface()) RegisterAPIInterface(NewEnvAPIInterface()) + RegisterAPIInterface(NewPipelineAPIInterface()) // Extension RegisterAPIInterface(NewDefinitionAPIInterface()) @@ -82,7 +83,6 @@ func InitAPIBean() []interface{} { RegisterAPIInterface(NewWebhookAPIInterface()) RegisterAPIInterface(NewRepositoryAPIInterface()) RegisterAPIInterface(NewCloudShellAPIInterface()) - RegisterAPIInterface(NewPipelineAPIInterface()) // Authentication RegisterAPIInterface(NewAuthenticationAPIInterface()) diff --git a/pkg/apiserver/interfaces/api/pipeline.go b/pkg/apiserver/interfaces/api/pipeline.go index 95e04847b..695a19fc4 100644 --- a/pkg/apiserver/interfaces/api/pipeline.go +++ b/pkg/apiserver/interfaces/api/pipeline.go @@ -18,199 +18,242 @@ package api import ( "context" + "strconv" restfulspec "github.com/emicklei/go-restful-openapi/v2" "github.com/emicklei/go-restful/v3" - workflowv1alpha1 "github.com/kubevela/workflow/api/v1alpha1" - - "github.com/oam-dev/kubevela/pkg/apiserver/utils/log" + "github.com/kubevela/workflow/api/v1alpha1" + "github.com/pkg/errors" "github.com/oam-dev/kubevela/pkg/apiserver/domain/service" apis "github.com/oam-dev/kubevela/pkg/apiserver/interfaces/api/dto/v1" "github.com/oam-dev/kubevela/pkg/apiserver/utils/bcode" + "github.com/oam-dev/kubevela/pkg/apiserver/utils/log" ) -type pipelineAPIInterface struct { - PipelineService service.PipelineService `inject:""` - PipelineRunService service.PipelineRunService `inject:""` - ContextService service.ContextService `inject:""` -} - -type pipelinePathParamKey string - const ( // Project is the project name key of query param - Project pipelinePathParamKey = "projectName" + Project string = "projectName" // Pipeline is the pipeline name of query param - Pipeline pipelinePathParamKey = "pipelineName" + Pipeline string = "pipelineName" // PipelineRun is the pipeline run name of query param - PipelineRun pipelinePathParamKey = "runName" + PipelineRun string = "runName" // ContextName is the context name of query param - ContextName pipelinePathParamKey = "contextName" + ContextName string = "contextName" ) -// GetWebServiceRoute is the implementation of pipeline Interface -func (p *pipelineAPIInterface) GetWebServiceRoute() *restful.WebService { - - ws := new(restful.WebService) +func initPipelineRoutes(ws *restful.WebService, n *projectAPIInterface) { tags := []string{"pipeline"} - projParam := func(builder *restful.RouteBuilder) { - builder.Param(ws.QueryParameter(string(Project), "project name").Required(true)) + builder.Param(ws.PathParameter(Project, "project name").Required(true)) + builder.Filter(n.projectCheckFilter) } pipelineParam := func(builder *restful.RouteBuilder) { - builder.Param(ws.PathParameter(string(Pipeline), "pipeline name").Required(true)) - builder.Filter(p.pipelineCheckFilter) + builder.Param(ws.PathParameter(Pipeline, "pipeline name").Required(true)) + builder.Filter(n.pipelineCheckFilter) } ctxParam := func(builder *restful.RouteBuilder) { - builder.Param(ws.PathParameter(string(ContextName), "pipeline context name").Required(true)) - builder.Filter(p.pipelineContextCheckFilter) + builder.Param(ws.PathParameter(ContextName, "pipeline context name").Required(true)) + builder.Filter(n.pipelineContextCheckFilter) } runParam := func(builder *restful.RouteBuilder) { - builder.Param(ws.PathParameter(string(PipelineRun), "pipeline run name").Required(true)) - builder.Filter(p.pipelineRunCheckFilter) + builder.Param(ws.PathParameter(PipelineRun, "pipeline run name").Required(true)) + builder.Filter(n.pipelineRunCheckFilter) } meta := func(builder *restful.RouteBuilder) { builder.Metadata(restfulspec.KeyOpenAPITags, tags) } - ws.Path(versionPrefix+"/pipelines"). - Consumes(restful.MIME_JSON, restful.MIME_XML). - Produces(restful.MIME_JSON, restful.MIME_XML). - Doc("api for pipeline manage") - - ws.Route(ws.POST("").To(p.createPipeline). + ws.Route(ws.POST("/{projectName}/pipelines").To(n.createPipeline). Doc("create pipeline"). Reads(apis.CreatePipelineRequest{}). Returns(200, "OK", apis.PipelineBase{}). Returns(400, "Bad Request", bcode.Bcode{}). - Writes(apis.PipelineBase{}).Do(meta)) + Filter(n.RBACService.CheckPerm("project/pipeline", "create")). + Writes(apis.PipelineBase{}).Do(meta, projParam)) - ws.Route(ws.GET("").To(p.listPipelines). - Doc("list pipelines"). - Param(ws.QueryParameter("query", "Fuzzy search based on name or description").DataType("string")). - Returns(200, "OK", apis.ListPipelineResponse{}). - Returns(400, "Bad Request", bcode.Bcode{}). - Writes(apis.ListPipelineResponse{}).Do(meta, projParam)) - - ws.Route(ws.GET("/{pipelineName}").To(p.getPipeline). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}").To(n.getPipeline). Doc("get pipeline"). - Reads(apis.GetPipelineRequest{}). Returns(200, "OK", apis.GetPipelineResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). - Writes(apis.GetPipelineResponse{}).Do(meta, projParam, pipelineParam)) + // use Param instead of pipelineParam to get pipeline information + Param(ws.PathParameter(Pipeline, "pipeline name").Required(true)). + Filter(n.RBACService.CheckPerm("project/pipeline", "detail")). + Writes(apis.GetPipelineResponse{}).Do(meta, projParam)) - ws.Route(ws.PUT("/{pipelineName}").To(p.updatePipeline). + ws.Route(ws.PUT("/{projectName}/pipelines/{pipelineName}").To(n.updatePipeline). Doc("update pipeline"). Reads(apis.UpdatePipelineRequest{}). Returns(200, "OK", apis.PipelineBase{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline", "update")). Writes(apis.PipelineBase{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.DELETE("/{pipelineName}").To(p.deletePipeline). + ws.Route(ws.DELETE("/{projectName}/pipelines/{pipelineName}").To(n.deletePipeline). Doc("delete pipeline"). Returns(200, "OK", apis.PipelineMetaResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline", "delete")). Writes(apis.PipelineMetaResponse{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.POST("/{pipelineName}/contexts").To(p.createContextValue). + ws.Route(ws.POST("/{projectName}/pipelines/{pipelineName}/contexts").To(n.createContextValue). Doc("create pipeline context values"). Reads(apis.CreateContextValuesRequest{}). Returns(200, "OK", apis.Context{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/context", "create")). Writes(apis.Context{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.GET("/{pipelineName}/contexts").To(p.listContextValues). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/contexts").To(n.listContextValues). Doc("list pipeline context values"). Returns(200, "OK", apis.ListContextValueResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/context", "list")). Writes(apis.ListContextValueResponse{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.PUT("/{pipelineName}/contexts/{contextName}").To(p.updateContextValue). + ws.Route(ws.PUT("/{projectName}/pipelines/{pipelineName}/contexts/{contextName}").To(n.updateContextValue). Doc("update pipeline context value"). Reads(apis.UpdateContextValuesRequest{}). Returns(200, "OK", apis.Context{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/context", "update")). Writes(apis.Context{}).Do(meta, projParam, pipelineParam, ctxParam)) - ws.Route(ws.DELETE("/{pipelineName}/contexts/{contextName}").To(p.deleteContextValue). + ws.Route(ws.DELETE("/{projectName}/pipelines/{pipelineName}/contexts/{contextName}").To(n.deleteContextValue). Doc("delete pipeline context value"). Returns(200, "OK", apis.ContextNameResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/context", "delete")). Writes(apis.ContextNameResponse{}).Do(meta, projParam, pipelineParam, ctxParam)) - ws.Route(ws.POST("/{pipelineName}/run").To(p.runPipeline). + ws.Route(ws.POST("/{projectName}/pipelines/{pipelineName}/run").To(n.runPipeline). Doc("run pipeline"). Reads(apis.RunPipelineRequest{}). - Returns(200, "OK", apis.PipelineRunMeta{}). + Returns(200, "OK", apis.PipelineRun{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline", "run")). Writes(apis.PipelineRunMeta{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.GET("/{pipelineName}/runs").To(p.listPipelineRuns). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs").To(n.listPipelineRuns). Doc("list pipeline runs"). Param(ws.QueryParameter("status", "query identifier of the status").DataType("string")). Returns(200, "OK", apis.ListPipelineRunResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "list")). Writes(apis.ListPipelineRunResponse{}).Do(meta, projParam, pipelineParam)) - ws.Route(ws.POST("/{pipelineName}/runs/{runName}/stop").To(p.stopPipeline). + ws.Route(ws.POST("/{projectName}/pipelines/{pipelineName}/runs/{runName}/stop").To(n.stopPipeline). Doc("stop pipeline run"). Returns(200, "OK", apis.PipelineRunMeta{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "stop")). Writes(apis.PipelineRunMeta{}).Do(meta, projParam, pipelineParam, runParam)) - ws.Route(ws.GET("/{pipelineName}/runs/{runName}").To(p.getPipelineRun). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs/{runName}").To(n.getPipelineRun). Doc("get pipeline run"). Returns(200, "OK", apis.PipelineRunBase{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "get")). Writes(apis.PipelineRunBase{}).Do(meta, projParam, pipelineParam, runParam)) - ws.Route(ws.DELETE("/{pipelineName}/runs/{runName}").To(p.deletePipelineRun). + ws.Route(ws.DELETE("/{projectName}/pipelines/{pipelineName}/runs/{runName}").To(n.deletePipelineRun). Doc("delete pipeline run"). Returns(200, "OK", apis.PipelineRunMeta{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "delete")). Writes(apis.PipelineRunMeta{}).Do(meta, projParam, pipelineParam, runParam)) // get pipeline run status - ws.Route(ws.GET("/{pipelineName}/runs/{runName}/status").To(p.getPipelineRunStatus). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs/{runName}/status").To(n.getPipelineRunStatus). Doc("get pipeline run status"). - Returns(200, "OK", workflowv1alpha1.WorkflowRunStatus{}). + Returns(200, "OK", v1alpha1.WorkflowRunStatus{}). Returns(400, "Bad Request", bcode.Bcode{}). - Writes(workflowv1alpha1.WorkflowRunStatus{}).Do(meta, projParam, pipelineParam, runParam)) + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "detail")). + Writes(v1alpha1.WorkflowRunStatus{}).Do(meta, projParam, pipelineParam, runParam)) // get pipeline run log - ws.Route(ws.GET("/{pipelineName}/runs/{runName}/log").To(p.getPipelineRunLog). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs/{runName}/log").To(n.getPipelineRunLog). Doc("get pipeline run log"). - Param(ws.QueryParameter("step", "query by specific id").DataType("string")). + Param(ws.QueryParameter("step", "query by specific step name").DataType("string")). Returns(200, "OK", apis.GetPipelineRunLogResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "detail")). Writes(apis.GetPipelineRunLogResponse{}).Do(meta, projParam, pipelineParam, runParam)) // get pipeline run output - ws.Route(ws.GET("/{pipelineName}/runs/{runName}/output").To(p.getPipelineRunOutput). + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs/{runName}/output").To(n.getPipelineRunOutput). Doc("get pipeline run output"). - Param(ws.QueryParameter("step", "query by specific id").DataType("string")). + Param(ws.QueryParameter("step", "query by specific step name").DataType("string").Required(true)). Returns(200, "OK", apis.GetPipelineRunOutputResponse{}). Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "detail")). Writes(apis.GetPipelineRunOutputResponse{}).Do(meta, projParam, pipelineParam, runParam)) + // get pipeline run input + ws.Route(ws.GET("/{projectName}/pipelines/{pipelineName}/runs/{runName}/input").To(n.getPipelineRunInput). + Doc("get pipeline run input"). + Param(ws.QueryParameter("step", "query by specific step name").DataType("string").Required(true)). + Returns(200, "OK", apis.GetPipelineRunInputResponse{}). + Returns(400, "Bad Request", bcode.Bcode{}). + Filter(n.RBACService.CheckPerm("project/pipeline/pipelineRun", "detail")). + Writes(apis.GetPipelineRunInputResponse{}).Do(meta, projParam, pipelineParam, runParam)) + + ws.Filter(authCheckFilter) +} + +// GetWebServiceRoute is the implementation of pipeline Interface +func (n *pipelineAPIInterface) GetWebServiceRoute() *restful.WebService { + tags := []string{"pipeline"} + meta := func(builder *restful.RouteBuilder) { + builder.Metadata(restfulspec.KeyOpenAPITags, tags) + } + + ws := new(restful.WebService) + + ws.Path(versionPrefix+"/pipelines"). + Consumes(restful.MIME_XML, restful.MIME_JSON). + Produces(restful.MIME_JSON, restful.MIME_XML). + Doc("api for project manage") + + ws.Route(ws.GET("").To(n.listPipelines). + Doc("list pipelines"). + Param(ws.QueryParameter("query", "Fuzzy search based on name or description").DataType("string")). + Param(ws.QueryParameter("projectName", "query pipelines within a project").DataType("string")). + Param(ws.QueryParameter("detailed", "query pipelines with detail").DataType("bool").DefaultValue("true")). + Returns(200, "OK", apis.ListPipelineResponse{}). + Returns(400, "Bad Request", bcode.Bcode{}). + Writes(apis.ListPipelineResponse{}).Do(meta)) + ws.Filter(authCheckFilter) return ws } +type pipelineAPIInterface struct { + PipelineService service.PipelineService `inject:""` +} + // NewPipelineAPIInterface new pipeline manage APIInterface func NewPipelineAPIInterface() Interface { return &pipelineAPIInterface{} } -func (p *pipelineAPIInterface) listPipelines(req *restful.Request, res *restful.Response) { - var projetNames []string - if req.QueryParameter("project") != "" { - projetNames = append(projetNames, req.QueryParameter("project")) +func (n *pipelineAPIInterface) listPipelines(req *restful.Request, res *restful.Response) { + var projectNames []string + if req.QueryParameter(Project) != "" { + projectNames = append(projectNames, req.QueryParameter(Project)) } - pipelines, err := p.PipelineService.ListPipelines(req.Request.Context(), apis.ListPipelineRequest{ - Projects: projetNames, + _detailed := req.QueryParameter("detailed") + if _detailed == "" { + _detailed = "true" + } + detailed, err := strconv.ParseBool(_detailed) + if err != nil { + bcode.ReturnError(req, res, errors.Wrap(err, "invalid detailed param")) + } + pipelines, err := n.PipelineService.ListPipelines(req.Request.Context(), apis.ListPipelineRequest{ + Projects: projectNames, Query: req.QueryParameter("query"), + Detailed: detailed, }) if err != nil { log.Logger.Errorf("list pipeline failure %s", err.Error()) @@ -223,15 +266,18 @@ func (p *pipelineAPIInterface) listPipelines(req *restful.Request, res *restful. } } -func (p *pipelineAPIInterface) getPipeline(req *restful.Request, res *restful.Response) { - pipeline := req.Request.Context().Value(apis.CtxKeyPipeline).(apis.PipelineBase) +func (n *projectAPIInterface) getPipeline(req *restful.Request, res *restful.Response) { + pipeline, err := n.PipelineService.GetPipeline(req.Request.Context(), req.PathParameter(Pipeline), true) + if err != nil { + return + } if err := res.WriteEntity(pipeline); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) createPipeline(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) createPipeline(req *restful.Request, res *restful.Response) { var createReq apis.CreatePipelineRequest if err := req.ReadEntity(&createReq); err != nil { bcode.ReturnError(req, res, err) @@ -241,19 +287,25 @@ func (p *pipelineAPIInterface) createPipeline(req *restful.Request, res *restful bcode.ReturnError(req, res, err) return } - pipelineBase, err := p.PipelineService.CreatePipeline(req.Request.Context(), createReq) + pipelineBase, err := n.PipelineService.CreatePipeline(req.Request.Context(), createReq) if err != nil { log.Logger.Errorf("create pipeline failure %s", err.Error()) bcode.ReturnError(req, res, err) return } + _, err = n.ContextService.InitContext(req.Request.Context(), pipelineBase.Project.Name, pipelineBase.Name) + if err != nil { + log.Logger.Errorf("init pipeline context failure: %s", err.Error()) + bcode.ReturnError(req, res, err) + return + } if err := res.WriteEntity(pipelineBase); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) updatePipeline(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) updatePipeline(req *restful.Request, res *restful.Response) { var updateReq apis.UpdatePipelineRequest if err := req.ReadEntity(&updateReq); err != nil { bcode.ReturnError(req, res, err) @@ -263,8 +315,8 @@ func (p *pipelineAPIInterface) updatePipeline(req *restful.Request, res *restful bcode.ReturnError(req, res, err) return } - base := req.Request.Context().Value(apis.CtxKeyPipeline).(apis.PipelineBase) - pipelineBase, err := p.PipelineService.UpdatePipeline(req.Request.Context(), base.Name, base.Project, updateReq) + pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) + pipelineBase, err := n.PipelineService.UpdatePipeline(req.Request.Context(), pipeline.Name, updateReq) if err != nil { log.Logger.Errorf("update pipeline failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -276,43 +328,43 @@ func (p *pipelineAPIInterface) updatePipeline(req *restful.Request, res *restful } } -func (p *pipelineAPIInterface) deletePipeline(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) deletePipeline(req *restful.Request, res *restful.Response) { pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) - err := p.PipelineService.DeletePipeline(req.Request.Context(), pipeline) + err := n.PipelineService.DeletePipeline(req.Request.Context(), pipeline) if err != nil { log.Logger.Errorf("delete pipeline failure %s", err.Error()) bcode.ReturnError(req, res, err) return } - if err := res.WriteEntity(pipeline.PipelineMeta); err != nil { + if err := res.WriteEntity(apis.EmptyResponse{}); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) runPipeline(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) runPipeline(req *restful.Request, res *restful.Response) { var runReq apis.RunPipelineRequest pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) - if err := req.ReadEntity(runReq); err != nil { + if err := req.ReadEntity(&runReq); err != nil { bcode.ReturnError(req, res, err) return } - err := p.PipelineService.RunPipeline(req.Request.Context(), pipeline, runReq) + run, err := n.PipelineService.RunPipeline(req.Request.Context(), pipeline, runReq) if err != nil { log.Logger.Errorf("run pipeline failure %s", err.Error()) bcode.ReturnError(req, res, err) return } - if err := res.WriteEntity(pipeline.PipelineMeta); err != nil { + if err := res.WriteEntity(run); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) stopPipeline(req *restful.Request, res *restful.Response) { - pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(apis.PipelineRun) - err := p.PipelineRunService.StopPipelineRun(req.Request.Context(), pipelineRun.PipelineRunBase) +func (n *projectAPIInterface) stopPipeline(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) + err := n.PipelineRunService.StopPipelineRun(req.Request.Context(), pipelineRun.PipelineRunBase) if err != nil { log.Logger.Errorf("stop pipeline failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -324,9 +376,9 @@ func (p *pipelineAPIInterface) stopPipeline(req *restful.Request, res *restful.R } } -func (p *pipelineAPIInterface) listPipelineRuns(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) listPipelineRuns(req *restful.Request, res *restful.Response) { pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) - pipelineRuns, err := p.PipelineRunService.ListPipelineRuns(req.Request.Context(), pipeline) + pipelineRuns, err := n.PipelineRunService.ListPipelineRuns(req.Request.Context(), pipeline) if err != nil { log.Logger.Errorf("list pipeline runs failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -338,33 +390,65 @@ func (p *pipelineAPIInterface) listPipelineRuns(req *restful.Request, res *restf } } -func (p *pipelineAPIInterface) getPipelineRun(req *restful.Request, res *restful.Response) { - pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(apis.PipelineRun) +func (n *projectAPIInterface) getPipelineRun(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) if err := res.WriteEntity(pipelineRun.PipelineRunBase); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) getPipelineRunStatus(req *restful.Request, res *restful.Response) { - pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(apis.PipelineRun) +func (n *projectAPIInterface) getPipelineRunStatus(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) if err := res.WriteEntity(pipelineRun.Status); err != nil { bcode.ReturnError(req, res, err) return } } -func (p *pipelineAPIInterface) getPipelineRunLog(req *restful.Request, res *restful.Response) { - +func (n *projectAPIInterface) getPipelineRunLog(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) + logs, err := n.PipelineRunService.GetPipelineRunLog(req.Request.Context(), *pipelineRun, req.QueryParameter("step")) + if err != nil { + log.Logger.Errorf("get pipeline run log failure %s", err.Error()) + bcode.ReturnError(req, res, err) + return + } + if err := res.WriteEntity(logs); err != nil { + bcode.ReturnError(req, res, err) + return + } } -func (p *pipelineAPIInterface) getPipelineRunOutput(req *restful.Request, res *restful.Response) { - +func (n *projectAPIInterface) getPipelineRunOutput(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) + output, err := n.PipelineRunService.GetPipelineRunOutput(req.Request.Context(), *pipelineRun, req.QueryParameter("step")) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + if err := res.WriteEntity(output); err != nil { + bcode.ReturnError(req, res, err) + return + } } -func (p *pipelineAPIInterface) deletePipelineRun(req *restful.Request, res *restful.Response) { - pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(apis.PipelineRun) - err := p.PipelineRunService.DeletePipelineRun(req.Request.Context(), pipelineRun.PipelineRunMeta) +func (n *projectAPIInterface) getPipelineRunInput(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) + input, err := n.PipelineRunService.GetPipelineRunInput(req.Request.Context(), *pipelineRun, req.QueryParameter("step")) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + if err := res.WriteEntity(input); err != nil { + bcode.ReturnError(req, res, err) + return + } +} + +func (n *projectAPIInterface) deletePipelineRun(req *restful.Request, res *restful.Response) { + pipelineRun := req.Request.Context().Value(&apis.CtxKeyPipelineRun).(*apis.PipelineRun) + err := n.PipelineRunService.DeletePipelineRun(req.Request.Context(), pipelineRun.PipelineRunMeta) if err != nil { log.Logger.Errorf("delete pipeline run failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -376,11 +460,11 @@ func (p *pipelineAPIInterface) deletePipelineRun(req *restful.Request, res *rest } } -func (p *pipelineAPIInterface) listContextValues(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) listContextValues(req *restful.Request, res *restful.Response) { pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) - contextValues, err := p.ContextService.ListContexts(req.Request.Context(), pipeline.Project, pipeline.Name) + contextValues, err := n.ContextService.ListContexts(req.Request.Context(), pipeline.Project.Name, pipeline.Name) if err != nil { - log.Logger.Errorf("list context values failure %s", err.Error()) + log.Logger.Errorf("list context values failure: %s", err.Error()) bcode.ReturnError(req, res, err) return } @@ -390,7 +474,7 @@ func (p *pipelineAPIInterface) listContextValues(req *restful.Request, res *rest } } -func (p *pipelineAPIInterface) createContextValue(req *restful.Request, res *restful.Response) { +func (n *projectAPIInterface) createContextValue(req *restful.Request, res *restful.Response) { pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) var createReq apis.CreateContextValuesRequest if err := req.ReadEntity(&createReq); err != nil { @@ -403,7 +487,7 @@ func (p *pipelineAPIInterface) createContextValue(req *restful.Request, res *res } pipelineCtx := apis.Context(createReq) - _, err := p.ContextService.CreateContext(req.Request.Context(), pipeline.Project, pipeline.Name, pipelineCtx) + _, err := n.ContextService.CreateContext(req.Request.Context(), pipeline.Project.Name, pipeline.Name, pipelineCtx) if err != nil { log.Logger.Errorf("create context failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -415,8 +499,8 @@ func (p *pipelineAPIInterface) createContextValue(req *restful.Request, res *res } } -func (p *pipelineAPIInterface) updateContextValue(req *restful.Request, res *restful.Response) { - plCtx := req.Request.Context().Value(&apis.CtxKeyPipelineContex).(apis.Context) +func (n *projectAPIInterface) updateContextValue(req *restful.Request, res *restful.Response) { + plCtx := req.Request.Context().Value(&apis.CtxKeyPipelineContext).(apis.Context) pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) var updateReq apis.UpdateContextValuesRequest if err := req.ReadEntity(&updateReq); err != nil { @@ -428,7 +512,7 @@ func (p *pipelineAPIInterface) updateContextValue(req *restful.Request, res *res return } pipelineCtx := apis.Context{Name: plCtx.Name, Values: updateReq.Values} - _, err := p.ContextService.UpdateContext(req.Request.Context(), pipeline.Project, pipeline.Name, pipelineCtx) + _, err := n.ContextService.UpdateContext(req.Request.Context(), pipeline.Project.Name, pipeline.Name, pipelineCtx) if err != nil { log.Logger.Errorf("update context failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -440,10 +524,10 @@ func (p *pipelineAPIInterface) updateContextValue(req *restful.Request, res *res } } -func (p *pipelineAPIInterface) deleteContextValue(req *restful.Request, res *restful.Response) { - plCtx := req.Request.Context().Value(&apis.CtxKeyPipelineContex).(apis.Context) +func (n *projectAPIInterface) deleteContextValue(req *restful.Request, res *restful.Response) { + plCtx := req.Request.Context().Value(&apis.CtxKeyPipelineContext).(apis.Context) pipeline := req.Request.Context().Value(&apis.CtxKeyPipeline).(apis.PipelineBase) - err := p.ContextService.DeleteContext(req.Request.Context(), pipeline.Project, pipeline.Name, plCtx.Name) + err := n.ContextService.DeleteContext(req.Request.Context(), pipeline.Project.Name, pipeline.Name, plCtx.Name) if err != nil { log.Logger.Errorf("delete context failure %s", err.Error()) bcode.ReturnError(req, res, err) @@ -455,8 +539,18 @@ func (p *pipelineAPIInterface) deleteContextValue(req *restful.Request, res *res } } -func (p *pipelineAPIInterface) pipelineCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { - pipeline, err := p.PipelineService.GetPipeline(req.Request.Context(), req.PathParameter("pipelineName"), req.QueryParameter("projectName")) +func (n *projectAPIInterface) projectCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { + project, err := n.ProjectService.GetProject(req.Request.Context(), req.PathParameter(Project)) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + req.Request = req.Request.WithContext(context.WithValue(req.Request.Context(), &apis.CtxKeyProject, project)) + chain.ProcessFilter(req, res) +} + +func (n *projectAPIInterface) pipelineCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { + pipeline, err := n.PipelineService.GetPipeline(req.Request.Context(), req.PathParameter(Pipeline), false) if err != nil { bcode.ReturnError(req, res, err) return @@ -465,8 +559,8 @@ func (p *pipelineAPIInterface) pipelineCheckFilter(req *restful.Request, res *re chain.ProcessFilter(req, res) } -func (p *pipelineAPIInterface) pipelineContextCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { - contexts, err := p.ContextService.ListContexts(req.Request.Context(), req.PathParameter("pipelineName"), req.QueryParameter("projectName")) +func (n *projectAPIInterface) pipelineContextCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { + contexts, err := n.ContextService.ListContexts(req.Request.Context(), req.PathParameter(Project), req.PathParameter(Pipeline)) if err != nil { bcode.ReturnError(req, res, err) return @@ -477,20 +571,22 @@ func (p *pipelineAPIInterface) pipelineContextCheckFilter(req *restful.Request, bcode.ReturnError(req, res, bcode.ErrContextNotFound) return } - req.Request = req.Request.WithContext(context.WithValue(req.Request.Context(), &apis.CtxKeyPipelineContex, apis.Context{ + req.Request = req.Request.WithContext(context.WithValue(req.Request.Context(), &apis.CtxKeyPipelineContext, apis.Context{ Name: contextName, Values: contextValue, })) chain.ProcessFilter(req, res) } -func (p *pipelineAPIInterface) pipelineRunCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { +func (n *projectAPIInterface) pipelineRunCheckFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { meta := apis.PipelineRunMeta{ - PipelineName: req.PathParameter(string(Pipeline)), - Project: req.QueryParameter(string(Project)), - PipelineRunName: req.PathParameter(string(PipelineRun)), + PipelineName: req.PathParameter(Pipeline), + Project: apis.NameAlias{ + Name: req.PathParameter(Project), + }, + PipelineRunName: req.PathParameter(PipelineRun), } - run, err := p.PipelineRunService.GetPipelineRun(req.Request.Context(), meta) + run, err := n.PipelineRunService.GetPipelineRun(req.Request.Context(), meta) if err != nil { bcode.ReturnError(req, res, err) return diff --git a/pkg/apiserver/interfaces/api/project.go b/pkg/apiserver/interfaces/api/project.go index 8a7c548de..032d5bda4 100644 --- a/pkg/apiserver/interfaces/api/project.go +++ b/pkg/apiserver/interfaces/api/project.go @@ -29,10 +29,14 @@ import ( ) type projectAPIInterface struct { - RbacService service.RBACService `inject:""` - ProjectService service.ProjectService `inject:""` - TargetService service.TargetService `inject:""` - ConfigService service.ConfigService `inject:""` + RbacService service.RBACService `inject:""` + ProjectService service.ProjectService `inject:""` + TargetService service.TargetService `inject:""` + ConfigService service.ConfigService `inject:""` + PipelineService service.PipelineService `inject:""` + PipelineRunService service.PipelineRunService `inject:""` + ContextService service.ContextService `inject:""` + RBACService service.RBACService `inject:""` } // NewProjectAPIInterface new project APIInterface @@ -305,6 +309,7 @@ func (n *projectAPIInterface) GetWebServiceRoute() *restful.WebService { Returns(400, "Bad Request", bcode.Bcode{}). Writes(apis.ListTerraformProviderResponse{})) + initPipelineRoutes(ws, n) ws.Filter(authCheckFilter) return ws } diff --git a/pkg/apiserver/utils/bcode/017_pipeline.go b/pkg/apiserver/utils/bcode/017_pipeline.go index 6b31f5ad6..635b0cf95 100644 --- a/pkg/apiserver/utils/bcode/017_pipeline.go +++ b/pkg/apiserver/utils/bcode/017_pipeline.go @@ -21,4 +21,22 @@ var ( ErrContextNotFound = NewBcode(400, 17001, "pipeline context is not found") // ErrContextAlreadyExist means the certain context already exists ErrContextAlreadyExist = NewBcode(400, 17002, "pipeline context of pipeline already exist") + // ErrGetPipelineInfo means failed to get pipeline info + ErrGetPipelineInfo = NewBcode(400, 17003, "get pipeline info failed") + // ErrPipelineNotExist means specific pipeline not found + ErrPipelineNotExist = NewBcode(404, 17004, "failed to find log pods") + // ErrGetPodsLogs means failed to get pods logs + ErrGetPodsLogs = NewBcode(500, 17006, "failed to get pods logs") + // ErrReadSourceLog means failed to read source log + ErrReadSourceLog = NewBcode(500, 17007, "failed to read log from URL source") + // ErrGetContextBackendData means failed to get context backend data + ErrGetContextBackendData = NewBcode(500, 17008, "failed to get context backend data") + // ErrNoSteps means pipeline doesn't have a step + ErrNoSteps = NewBcode(400, 17009, "pipeline step number is zero") + // ErrPipelineExist means the pipeline is exist + ErrPipelineExist = NewBcode(400, 17010, "the pipeline is exist") + // ErrPipelineRunFinished means pipeline run is finished + ErrPipelineRunFinished = NewBcode(400, 17011, "pipeline run is finished") + // ErrWrongMode means the pipeline run mode is wrong + ErrWrongMode = NewBcode(400, 17012, "wrong pipeline run mode, only \"DAG\" and \"StepByStep\" are supported") ) diff --git a/pkg/utils/k8s.go b/pkg/utils/k8s.go index 4eba13c82..948ffab68 100644 --- a/pkg/utils/k8s.go +++ b/pkg/utils/k8s.go @@ -21,7 +21,19 @@ import ( "encoding/json" "fmt" "os" + "regexp" "strings" + "text/template" + "time" + + "github.com/fatih/color" + "github.com/pkg/errors" + "github.com/wercker/stern/stern" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/client-go/kubernetes" + + querytypes "github.com/oam-dev/kubevela/pkg/velaql/providers/query/types" authv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" @@ -199,3 +211,125 @@ func IsClusterScope(gvk schema.GroupVersionKind, mapper meta.RESTMapper) (bool, isClusterScope := len(mappings) > 0 && mappings[0].Scope.Name() == meta.RESTScopeNameRoot return isClusterScope, err } + +// GetPodsLogs get logs from pods +func GetPodsLogs(ctx context.Context, config *rest.Config, containerName string, selectPods []*querytypes.PodBase, tmpl string, logC chan<- string, tailLines *int64) error { + if err := verifyPods(selectPods); err != nil { + return err + } + podRegex := getPodRegex(selectPods) + pods, err := regexp.Compile(podRegex) + if err != nil { + return fmt.Errorf("fail to compile '%s' for logs query", podRegex) + } + container := regexp.MustCompile(".*") + if containerName != "" { + container = regexp.MustCompile(containerName + ".*") + } + // These pods are from the same namespace, so we can use the first one to get the namespace + namespace := selectPods[0].Metadata.Namespace + selector := labels.NewSelector() + // Only use the labels to select pod if query one pod's log. It is only used when query vela-core log + if len(selectPods) == 1 { + for k, v := range selectPods[0].Metadata.Labels { + req, _ := labels.NewRequirement(k, selection.Equals, []string{v}) + if req != nil { + selector = selector.Add(*req) + } + } + } + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + return err + } + added, removed, err := stern.Watch(ctx, + clientSet.CoreV1().Pods(namespace), + pods, + container, + nil, + []stern.ContainerState{stern.RUNNING, stern.TERMINATED}, + selector, + ) + if err != nil { + return err + } + tails := make(map[string]*stern.Tail) + + funs := map[string]interface{}{ + "json": func(in interface{}) (string, error) { + b, err := json.Marshal(in) + if err != nil { + return "", err + } + return string(b), nil + }, + "color": func(color color.Color, text string) string { + return color.SprintFunc()(text) + }, + } + template, err := template.New("log").Funcs(funs).Parse(tmpl) + if err != nil { + return errors.Wrap(err, "unable to parse template") + } + + go func() { + for p := range added { + id := p.GetID() + if tails[id] != nil { + continue + } + // 48h + dur, _ := time.ParseDuration("48h") + tail := stern.NewTail(p.Namespace, p.Pod, p.Container, template, &stern.TailOptions{ + Timestamps: true, + SinceSeconds: int64(dur.Seconds()), + Exclude: nil, + Include: nil, + Namespace: false, + TailLines: tailLines, // default for all logs + }) + tails[id] = tail + + tail.Start(ctx, clientSet.CoreV1().Pods(p.Namespace), logC) + } + }() + + go func() { + for p := range removed { + id := p.GetID() + if tails[id] == nil { + continue + } + tails[id].Close() + delete(tails, id) + } + }() + + <-ctx.Done() + close(logC) + return nil +} + +func getPodRegex(pods []*querytypes.PodBase) string { + var podNames []string + for _, pod := range pods { + podNames = append(podNames, fmt.Sprintf("(%s.*)", pod.Metadata.Name)) + } + return strings.Join(podNames, "|") +} + +func verifyPods(pods []*querytypes.PodBase) error { + if len(pods) == 0 { + return errors.New("no pods selected") + } + if len(pods) == 1 { + return nil + } + namespace := pods[0].Metadata.Namespace + for _, pod := range pods { + if pod.Metadata.Namespace != namespace { + return errors.New("cannot select pods from different namespaces") + } + } + return nil +} diff --git a/references/cli/logs.go b/references/cli/logs.go index d1b012eea..1fd5d23ba 100644 --- a/references/cli/logs.go +++ b/references/cli/logs.go @@ -18,24 +18,16 @@ package cli import ( "context" - "encoding/json" "fmt" - "regexp" "strings" - "text/template" - "time" "github.com/fatih/color" - "github.com/pkg/errors" "github.com/spf13/cobra" - "github.com/wercker/stern/stern" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/selection" - "k8s.io/client-go/kubernetes" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/multicluster" + "github.com/oam-dev/kubevela/pkg/utils" "github.com/oam-dev/kubevela/pkg/utils/common" "github.com/oam-dev/kubevela/pkg/utils/util" querytypes "github.com/oam-dev/kubevela/pkg/velaql/providers/query/types" @@ -99,45 +91,25 @@ type Args struct { } func (l *Args) printPodLogs(ctx context.Context, ioStreams util.IOStreams, selectPod *querytypes.PodBase, filters []string) error { - pod, err := regexp.Compile(selectPod.Metadata.Name + ".*") - if err != nil { - return fmt.Errorf("fail to compile '%s' for logs query", selectPod.Metadata.Name+".*") - } - container := regexp.MustCompile(".*") - if l.ContainerName != "" { - container = regexp.MustCompile(l.ContainerName + ".*") - } - namespace := selectPod.Metadata.Namespace - selector := labels.NewSelector() - for k, v := range selectPod.Metadata.Labels { - req, _ := labels.NewRequirement(k, selection.Equals, []string{v}) - if req != nil { - selector = selector.Add(*req) - } - } - config, err := l.Args.GetConfig() if err != nil { return err } - clientSet, err := kubernetes.NewForConfig(config) - if err != nil { - return err - } - added, removed, err := stern.Watch(ctx, - clientSet.CoreV1().Pods(namespace), - pod, - container, - nil, - []stern.ContainerState{stern.RUNNING, stern.TERMINATED}, - selector, - ) - if err != nil { - return err - } - tails := make(map[string]*stern.Tail) logC := make(chan string, 1024) + var t string + switch l.Output { + case "default": + if color.NoColor { + t = "{{.ContainerName}} {{.Message}}" + } else { + t = "{{color .ContainerColor .ContainerName}} {{.Message}}" + } + case "raw": + t = "{{.Message}}" + case "json": + t = "{{json .}}\n" + } go func() { for { select { @@ -158,71 +130,11 @@ func (l *Args) printPodLogs(ctx context.Context, ioStreams util.IOStreams, selec } }() - var t string - switch l.Output { - case "default": - if color.NoColor { - t = "{{.ContainerName}} {{.Message}}" - } else { - t = "{{color .ContainerColor .ContainerName}} {{.Message}}" - } - case "raw": - t = "{{.Message}}" - case "json": - t = "{{json .}}\n" - } - funs := map[string]interface{}{ - "json": func(in interface{}) (string, error) { - b, err := json.Marshal(in) - if err != nil { - return "", err - } - return string(b), nil - }, - "color": func(color color.Color, text string) string { - return color.SprintFunc()(text) - }, - } - template, err := template.New("log").Funcs(funs).Parse(t) + err = utils.GetPodsLogs(ctx, config, l.ContainerName, []*querytypes.PodBase{selectPod}, t, logC, nil) if err != nil { - return errors.Wrap(err, "unable to parse template") + return err } - go func() { - for p := range added { - id := p.GetID() - if tails[id] != nil { - continue - } - // 48h - dur, _ := time.ParseDuration("48h") - tail := stern.NewTail(p.Namespace, p.Pod, p.Container, template, &stern.TailOptions{ - Timestamps: true, - SinceSeconds: int64(dur.Seconds()), - Exclude: nil, - Include: nil, - Namespace: false, - TailLines: nil, // default for all logs - }) - tails[id] = tail - - tail.Start(ctx, clientSet.CoreV1().Pods(p.Namespace), logC) - } - }() - - go func() { - for p := range removed { - id := p.GetID() - if tails[id] == nil { - continue - } - tails[id].Close() - delete(tails, id) - } - }() - - <-ctx.Done() - return nil } diff --git a/test/e2e-apiserver-test/pipeline_test.go b/test/e2e-apiserver-test/pipeline_test.go new file mode 100644 index 000000000..de5b68bc7 --- /dev/null +++ b/test/e2e-apiserver-test/pipeline_test.go @@ -0,0 +1,363 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_apiserver_test + +import ( + "context" + "net/http" + "strconv" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/kubevela/workflow/api/v1alpha1" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + "github.com/oam-dev/kubevela/pkg/apiserver/domain/model" + apisv1 "github.com/oam-dev/kubevela/pkg/apiserver/interfaces/api/dto/v1" + "github.com/oam-dev/kubevela/pkg/oam/util" + "github.com/oam-dev/kubevela/pkg/utils/common" +) + +var testPipelineSteps []v1alpha1.WorkflowStep + +func init() { + rawProps := []byte(`{"url":"https://api.github.com/repos/kubevela/kubevela"}`) + testPipelineSteps = []v1alpha1.WorkflowStep{ + { + WorkflowStepBase: v1alpha1.WorkflowStepBase{ + Name: "request", + Type: "request", + Outputs: v1alpha1.StepOutputs{ + { + ValueFrom: "import \"strconv\"\n\"Current star count: \" + strconv.FormatInt(response[\"stargazers_count\"], 10)\n", + Name: "stars", + }, + }, + Properties: &runtime.RawExtension{ + Raw: rawProps, + }, + }, + }, + } +} + +var _ = Describe("Test the rest api about the pipeline", func() { + var ( + projectName1 = testNSprefix + strconv.FormatInt(time.Now().UnixNano(), 10) + pipelineName = "test-pipeline" + description = "amazing pipeline" + contextName = "test-context" + contextKey = "test-key" + contextVal = "test-val" + pipelineRunName string + ) + defer GinkgoRecover() + It("create project and apply definitions", func() { + defer GinkgoRecover() + var req = apisv1.CreateProjectRequest{ + Name: projectName1, + Description: "KubeVela Project", + } + res := post("/projects", req) + var projectBase apisv1.ProjectBase + Expect(decodeResponseBody(res, &projectBase)).Should(Succeed()) + Expect(cmp.Diff(projectBase.Name, req.Name)).Should(BeEmpty()) + Expect(cmp.Diff(projectBase.Description, req.Description)).Should(BeEmpty()) + + def1 := new(v1beta1.WorkflowStepDefinition) + def2 := new(v1beta1.WorkflowStepDefinition) + Expect(common.ReadYamlToObject("./testdata/request.yaml", def1)).Should(BeNil()) + Expect(k8sClient.Create(context.Background(), def1)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + Expect(common.ReadYamlToObject("./testdata/log.yaml", def2)).Should(BeNil()) + Expect(k8sClient.Create(context.Background(), def2)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + }) + + It("create pipeline", func() { + var req = apisv1.CreatePipelineRequest{ + Name: pipelineName, + Description: description, + Spec: v1alpha1.WorkflowSpec{ + Steps: testPipelineSteps, + }, + } + res := post("/projects/"+projectName1+"/pipelines", req) + var pipeline apisv1.PipelineBase + Expect(decodeResponseBody(res, &pipeline)).Should(Succeed()) + Expect(cmp.Diff(pipeline.Name, req.Name)).Should(BeEmpty()) + Expect(len(pipeline.Spec.Steps)).Should(Equal(len(req.Spec.Steps))) + }) + + It("create context", func() { + var req = apisv1.CreateContextValuesRequest{ + Name: contextName, + Values: []model.Value{ + { + Key: contextKey, + Value: contextVal, + }, + }, + } + res := post("/projects/"+projectName1+"/pipelines/"+pipelineName+"/contexts", req) + var context apisv1.Context + Expect(decodeResponseBody(res, &context)).Should(Succeed()) + Expect(cmp.Diff(context.Name, req.Name)).Should(BeEmpty()) + Expect(cmp.Diff(context.Values, req.Values)).Should(BeEmpty()) + }) + + It("get contexts", func() { + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/contexts") + var contexs apisv1.ListContextValueResponse + Expect(decodeResponseBody(res, &contexs)).Should(Succeed()) + Expect(len(contexs.Contexts)).Should(Equal(1)) + ctx, ok := contexs.Contexts[contextName] + Expect(ok).Should(BeTrue()) + Expect(len(ctx)).Should(Equal(1)) + }) + + It("update context", func() { + var req = apisv1.UpdateContextValuesRequest{ + Values: []model.Value{ + { + Key: contextKey, + Value: "new-val", + }, + }, + } + res := put("/projects/"+projectName1+"/pipelines/"+pipelineName+"/contexts/"+contextName, req) + var context apisv1.Context + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + Expect(decodeResponseBody(res, &context)).Should(Succeed()) + + By("check the context value") + Expect(cmp.Diff(context.Values[0].Value, "new-val")).Should(BeEmpty()) + }) + + It("update pipeline", func() { + rawProps := []byte(`{"url":"https://api.github.com/repos/kubevela/kubevela"}`) + newSteps := make([]v1alpha1.WorkflowStep, 0) + newSteps = append(newSteps, v1alpha1.WorkflowStep{ + SubSteps: []v1alpha1.WorkflowStepBase{ + { + Name: "request1", + Type: "request", + Outputs: v1alpha1.StepOutputs{ + { + ValueFrom: "import \"strconv\"\n\"Current star count: \" + strconv.FormatInt(response[\"stargazers_count\"], 10)\n", + Name: "stars", + }, + }, + Properties: &runtime.RawExtension{ + Raw: rawProps, + }, + }, + { + Name: "request2", + Type: "request", + Outputs: v1alpha1.StepOutputs{ + { + ValueFrom: "import \"strconv\"\n\"Current star count: \" + strconv.FormatInt(response[\"stargazers_count\"], 10)\n", + Name: "stars-copy", + }, + }, + Properties: &runtime.RawExtension{ + Raw: rawProps, + }, + }, + }, + WorkflowStepBase: v1alpha1.WorkflowStepBase{ + Name: "request-group", + Type: "step-group", + }, + }) + newSteps = append(newSteps, v1alpha1.WorkflowStep{ + WorkflowStepBase: v1alpha1.WorkflowStepBase{ + Name: "log", + Type: "log", + Inputs: v1alpha1.StepInputs{ + { + ParameterKey: "data", + From: "stars", + }, + }, + }, + }) + var req = apisv1.UpdatePipelineRequest{ + Description: description, + Spec: v1alpha1.WorkflowSpec{ + Steps: newSteps, + }, + } + res := put("/projects/"+projectName1+"/pipelines/"+pipelineName, req) + var pipeline apisv1.PipelineBase + Expect(decodeResponseBody(res, &pipeline)).Should(Succeed()) + Expect(len(pipeline.Spec.Steps)).Should(Equal(len(req.Spec.Steps))) + }) + + It("run pipeline", func() { + var req = apisv1.RunPipelineRequest{ + Mode: v1alpha1.WorkflowExecuteMode{ + Steps: "StepByStep", + SubSteps: "DAG", + }, + ContextName: contextName, + } + res := post("/projects/"+projectName1+"/pipelines/"+pipelineName+"/run", req) + var run apisv1.PipelineRun + Expect(decodeResponseBody(res, &run)).Should(Succeed()) + Expect(run.PipelineRunName).ShouldNot(BeEmpty()) + pipelineRunName = run.PipelineRunName + }) + + It("list pipeline", func() { + res := get("/pipelines?query=amazing") + var pipelines apisv1.ListPipelineResponse + Expect(decodeResponseBody(res, &pipelines)).Should(Succeed()) + Expect(pipelines.Total).Should(BeNumerically("==", 1)) + Expect(pipelines.Pipelines[0].Name).Should(Equal(pipelineName)) + }) + + It("get pipeline", func() { + Eventually(func(g Gomega) { + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName) + var pipeline apisv1.GetPipelineResponse + g.Expect(decodeResponseBody(res, &pipeline)).Should(Succeed()) + g.Expect(pipeline.Name).Should(Equal(pipelineName)) + g.Expect(pipeline.Description).Should(Equal(description)) + g.Expect(pipeline.PipelineInfo.LastRun).ShouldNot(BeNil()) + g.Expect(pipeline.PipelineInfo.RunStat.Total).Should(Equal(apisv1.RunStatInfo{Total: 1, Success: 1})) + g.Expect(len(pipeline.PipelineInfo.RunStat.Week)).Should(Equal(7)) + }, 10*time.Second, 1*time.Second).Should(Succeed()) + }) + + It("list pipeline runs", func() { + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs") + var runs apisv1.ListPipelineRunResponse + Expect(decodeResponseBody(res, &runs)).Should(Succeed()) + Expect(runs.Total).Should(BeNumerically("==", 1)) + }) + + It("get pipeline run", func() { + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName) + var run apisv1.PipelineRunBase + Expect(decodeResponseBody(res, &run)).Should(Succeed()) + Expect(run.PipelineRunName).Should(Equal(pipelineRunName)) + }) + + It("get pipeline run status", func() { + Eventually(func(g Gomega) { + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName + "/status") + var status v1alpha1.WorkflowRunStatus + g.Expect(decodeResponseBody(res, &status)).Should(Succeed()) + g.Expect(status.Finished).Should(Equal(true)) + g.Expect(status.Phase).Should(Equal(v1alpha1.WorkflowStateSucceeded)) + g.Expect(status.Message).Should(BeEmpty()) + }, 100*time.Second, 1*time.Second).Should(Succeed()) + }) + + It("get pipeline run output", func() { + outputStep := "request1" + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName + "/output?step=" + outputStep) + var output apisv1.GetPipelineRunOutputResponse + Expect(decodeResponseBody(res, &output)).Should(Succeed()) + Expect(output.StepOutputs).Should(HaveLen(1)) + Expect(output.StepOutputs[0].Name).Should(Equal(outputStep)) + Expect(output.StepOutputs[0].Values).Should(HaveLen(1)) + Expect(output.StepOutputs[0].Values[0].Value).ShouldNot(BeEmpty()) + }) + + It("get pipeline run input", func() { + inputStep := "log" + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName + "/input?step=" + inputStep) + var input apisv1.GetPipelineRunInputResponse + Expect(decodeResponseBody(res, &input)).Should(Succeed()) + Expect(input.StepInputs).Should(HaveLen(1)) + Expect(input.StepInputs[0].Name).Should(Equal(inputStep)) + Expect(input.StepInputs[0].Values).Should(HaveLen(1)) + Expect(input.StepInputs[0].Values[0].Value).ShouldNot(BeEmpty()) + }) + + It("get pipeline run logs", func() { + logStep := "log" + res := get("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName + "/log?step=" + logStep) + var logs apisv1.GetPipelineRunLogResponse + Expect(decodeResponseBody(res, &logs)).Should(Succeed()) + Expect(logs.Name).Should(Equal(logStep)) + Expect(logs.Log).ShouldNot(BeEmpty()) + }) + + It("delete pipeline run", func() { + res := delete("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + }) + + It("stop pipeline", func() { + By("update pipeline so that it will run for a while") + var req = apisv1.UpdatePipelineRequest{ + Spec: v1alpha1.WorkflowSpec{ + Steps: []v1alpha1.WorkflowStep{ + { + WorkflowStepBase: v1alpha1.WorkflowStepBase{ + Name: "request", + Type: "request", + Timeout: "20s", + DependsOn: []string{"not-exist-step"}, + }, + }, + }, + }, + } + res := put("/projects/"+projectName1+"/pipelines/"+pipelineName, req) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + + By("run the pipeline") + var run apisv1.PipelineRun + res = post("/projects/"+projectName1+"/pipelines/"+pipelineName+"/run", apisv1.RunPipelineRequest{}) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + Expect(decodeResponseBody(res, &run)).Should(Succeed()) + pipelineRunName = run.PipelineRunName + + By("stop the pipeline") + var meta apisv1.PipelineRunMeta + res = post("/projects/"+projectName1+"/pipelines/"+pipelineName+"/runs/"+pipelineRunName+"/stop", nil) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + Expect(decodeResponseBody(res, &meta)).Should(Succeed()) + Expect(meta.PipelineRunName).Should(Equal(pipelineRunName)) + + By("delete pipeline run") + res = delete("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/runs/" + pipelineRunName) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + }) + + It("delete context", func() { + res := delete("/projects/" + projectName1 + "/pipelines/" + pipelineName + "/contexts/" + contextName) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + }) + + It("delete pipeline", func() { + res := delete("/projects/" + projectName1 + "/pipelines/" + pipelineName) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + }) + + It("delete project", func() { + res := delete("/projects/" + projectName1) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + }) + +}) diff --git a/test/e2e-apiserver-test/suite_test.go b/test/e2e-apiserver-test/suite_test.go index 4f1eb3392..1aff444b3 100644 --- a/test/e2e-apiserver-test/suite_test.go +++ b/test/e2e-apiserver-test/suite_test.go @@ -59,6 +59,7 @@ func TestE2eApiserverTest(t *testing.T) { // Suite test in e2e-apiserver-test relies on the pre-setup kubernetes environment var _ = BeforeSuite(func() { + defer GinkgoRecover() ctx := context.Background() @@ -115,7 +116,7 @@ var _ = BeforeSuite(func() { err = json.NewDecoder(resp.Body).Decode(code) Expect(err).Should(BeNil()) return fmt.Errorf("rest service not ready code:%d message:%s", resp.StatusCode, code.Message) - }, time.Second*10, time.Millisecond*200).Should(BeNil()) + }, time.Second*20, time.Millisecond*200).Should(BeNil()) var err error k8sClient, err = clients.GetKubeClient() Expect(err).ShouldNot(HaveOccurred()) diff --git a/test/e2e-apiserver-test/testdata/log.yaml b/test/e2e-apiserver-test/testdata/log.yaml new file mode 100644 index 000000000..b646796cb --- /dev/null +++ b/test/e2e-apiserver-test/testdata/log.yaml @@ -0,0 +1,35 @@ +apiVersion: core.oam.dev/v1beta1 +kind: WorkflowStepDefinition +metadata: + annotations: + definition.oam.dev/alias: "" + definition.oam.dev/description: Apply raw kubernetes objects for your workflow steps + labels: + custom.definition.oam.dev/ui-hidden: "true" + name: log + namespace: vela-system +spec: + schematic: + cue: + template: | + import ( + "vela/op" + ) + + apply: op.#Log & { + parameter + } + parameter: { + data?: string + level: *3 | int + source?: close({ + url: string + }) | close({ + resources?: [...{ + name?: string + cluster?: string + namespace?: string + labelSelector?: {...} + }] + }) + } diff --git a/test/e2e-apiserver-test/testdata/request.yaml b/test/e2e-apiserver-test/testdata/request.yaml new file mode 100644 index 000000000..2aa294d28 --- /dev/null +++ b/test/e2e-apiserver-test/testdata/request.yaml @@ -0,0 +1,43 @@ +apiVersion: core.oam.dev/v1beta1 +kind: WorkflowStepDefinition +metadata: + annotations: + definition.oam.dev/alias: "" + definition.oam.dev/description: Send request to the url + name: request + namespace: vela-system +spec: + schematic: + cue: + template: | + import ( + "vela/op" + "encoding/json" + ) + + http: op.http.#Do & { + method: parameter.method + url: parameter.url + request: { + if parameter.body != _|_ { + body: json.Marshal(parameter.body) + } + if parameter.header != _|_ { + header: parameter.header + } + } + } + fail: op.#Steps & { + if http.response.statusCode > 400 { + requestFail: op.#Fail & { + message: "request of \(parameter.url) is fail: \(http.response.statusCode)" + } + } + } + response: json.Unmarshal(http.response.body) + parameter: { + url: string + method: *"GET" | "POST" | "PUT" | "DELETE" + body?: {...} + header?: [string]: string + }