From 1776859631d62573100a9de52a2e844582f2d85a Mon Sep 17 00:00:00 2001 From: "Jian.Li" <74582607+leejanee@users.noreply.github.com> Date: Thu, 2 Sep 2021 13:40:10 +0800 Subject: [PATCH] Feat(workflow): Add op.#Task action (#2220) * Feat(workflow): tekton task * Feat(tools): random string * Fix(example): defintion fmt * Fix(rand): gosec weak random --- .../source-to-image/step-definition.yaml | 55 +++++++ pkg/stdlib/op.cue | 67 ++++---- pkg/stdlib/pkgs/task.cue | 155 ++++++++++++++++++ pkg/utils/random.go | 34 ++++ pkg/utils/util_test.go | 35 ++++ 5 files changed, 314 insertions(+), 32 deletions(-) create mode 100644 docs/examples/workflow/source-to-image/step-definition.yaml create mode 100644 pkg/stdlib/pkgs/task.cue create mode 100644 pkg/utils/random.go create mode 100644 pkg/utils/util_test.go diff --git a/docs/examples/workflow/source-to-image/step-definition.yaml b/docs/examples/workflow/source-to-image/step-definition.yaml new file mode 100644 index 000000000..d62a4f807 --- /dev/null +++ b/docs/examples/workflow/source-to-image/step-definition.yaml @@ -0,0 +1,55 @@ +apiVersion: core.oam.dev/v1beta1 +kind: WorkflowStepDefinition +metadata: + name: build + namespace: vela-system +spec: + schematic: + cue: + template: |- + import ("vela/op") + + parameter: { + repoUrl: string + dockerfile: string + image: string + } + + let pushImage=image + + build: op.#Task & { + name: "\(context.name)-\(context.stepId)" + namespace: context.namespace + + // Declare workspaces that used to share data. + workspace: { + "code-dir": {} + } + + steps: [ + // git clone code repo. + { + name: "git-clone" + image: "git-image-with-pull-ssh" + workspaceMounts: [{workspace: workspace["code-dir"], mountPath: "/data/code"}] + script: """ + #! /bin/bash + git clone \(repourl) /data/code + """ + }, + // build and push image. + { + name: "build-image" + image: "gcr.io/kaniko-project/executor:v1.3.0" + workspaceMounts: [{workspace: workspace["code-dir"], mountPath: "/data/build"}] + script: """ + #! /bin/bash + /kaniko/executor --dockerfile=/data/build/\(dockerfile) --context=/data/build --destination=\(pushImage) + """ + }] + } + + // wait until deployment ready + wait: op.#ConditionalWait & { + continue: build.do.status.phase == "successed" + } diff --git a/pkg/stdlib/op.cue b/pkg/stdlib/op.cue index 0642297ab..d534c047b 100644 --- a/pkg/stdlib/op.cue +++ b/pkg/stdlib/op.cue @@ -1,6 +1,8 @@ import ( "encoding/yaml" "encoding/json" + "encoding/base64" + "strings" ) #ConditionalWait: { @@ -16,13 +18,13 @@ import ( #Apply: kube.#Apply #ApplyApplication: #Steps & { - load: ws.#Load @step(1) + load: ws.#Load @step(1) components: #Steps & { for name, c in load.value { "\(name)": #Steps & { workload: kube.#Apply & {value: c.workload} if c.auxiliaries != _|_ { - _key: "trait.oam.dev/resource" + _key: "trait.oam.dev/resource" for index, o in c.auxiliaries { "\(o.metadata.labels[_key])": kube.#Apply & {value: o} } @@ -34,35 +36,34 @@ import ( #ApplyComponent: #Steps & { - component: string - _componentName: component - load: ws.#Load & { - component: _componentName - } @step(1) + component: string + _componentName: component + load: ws.#Load & { + component: _componentName + } @step(1) - traits: #Steps & { - _key: "trait.oam.dev/resource" - _manWlKey: "trait.oam.dev/manage-workload" - skipApplyWorkload: *false | bool - if load.value.auxiliaries != _|_ { - for o in load.value.auxiliaries { - "\(o.metadata.labels[_key])": kube.#Apply & {value: o} - if o.metadata.labels[_manWlKey] != _|_ { - skipApplyWorkload: true - } - } - } - } @step(2) - - workload__: { - if !traits.skipApplyWorkload { - kube.#Apply & { - value: load.value.workload - ... - } - } - } @step(3) + traits: #Steps & { + _key: "trait.oam.dev/resource" + _manWlKey: "trait.oam.dev/manage-workload" + skipApplyWorkload: *false | bool + if load.value.auxiliaries != _|_ { + for o in load.value.auxiliaries { + "\(o.metadata.labels[_key])": kube.#Apply & {value: o} + if o.metadata.labels[_manWlKey] != _|_ { + skipApplyWorkload: true + } + } + } + } @step(2) + workload__: { + if !traits.skipApplyWorkload { + kube.#Apply & { + value: load.value.workload + ... + } + } + } @step(3) } #ApplyRemaining: #Steps & { @@ -110,9 +111,9 @@ import ( } #DingTalk: #Steps & { - message: dingDing.#DingMessage - dingUrl: string - do: http.#Do & { + message: dingDing.#DingMessage + dingUrl: string + do: http.#Do & { method: "POST" url: dingUrl request: { @@ -194,4 +195,6 @@ import ( ... } +#Task: task.#Task + NoExist: _|_ diff --git a/pkg/stdlib/pkgs/task.cue b/pkg/stdlib/pkgs/task.cue new file mode 100644 index 000000000..ae16d8e26 --- /dev/null +++ b/pkg/stdlib/pkgs/task.cue @@ -0,0 +1,155 @@ + +let toolImage = "gcr.io/tekton-releases/github.com/tektoncd/pipeline/cmd/entrypoint:v0.27.2" +let baseImage = "gcr.io/distroless/base@sha256:aa4fd987555ea10e1a4ec8765da8158b5ffdfef1e72da512c7ede509bc9966c4" + +#Task: { + #do: "steps" + name: string + namespace: string + workspace: [_name_=string]: #workspace & {name: "\(_name_)"} + steps: [...#Script] + + _name: name + _namespace: namespace + _generate_scripts: [ for i, x in steps {""" +scriptfile="/vela/scripts/script-\(i)" +touch ${scriptfile} && chmod +x ${scriptfile} +cat > ${scriptfile} << '_EOF_' +\(base64.Encode(null, x.script)) +_EOF_ +"""}] + do: { + value: #PodTask & { + _scripts: strings.Join(_generate_scripts, "") + _workspace: workspace + metadata: { + name: _name + namespace: _namespace + } + spec: containers: [ for i, step in steps { + #StepContainer + name: step.name + image: step.image + env: step.envs + _workspaceMounts: workspace + _index: i + }] + } + } +} + +#Script: { + name: string + image: string + script: string + envs: [...{name: string, value: string}] + workspaceMounts: [...{workspace: #workspace, mountPath: string}] +} + +#workspace: { + name: string +} + +#PodTask: { + _scripts: string + _workspace: {...} + _volumes: [ for x in _workspace {name: x.name, emptyDir: {}}] + apiVersion: "v1" + kind: "Pod" + metadata: { + annotations: "vela.dev/ready": "READY" + namespace: *"default" | string + name: string + } + spec: { + containers: [...#StepContainer] + initContainers: [ + { + name: "place-tools" + command: ["/ko-app/entrypoint", "cp", "/ko-app/entrypoint", "/vela/tools/entrypoint"] + image: toolImage + imagePullPolicy: "IfNotPresent" + volumeMounts: [{name: "vela-internal-tools", mountPath: "/vela/tools"}] + }, { + name: "place-scripts" + imagePullPolicy: "IfNotPresent" + image: baseImage + command: ["sh"] + args: ["-c", _scripts + "\n/vela/tools/entrypoint decode-script \"${scriptfile}\""] + volumeMounts: [{name: "vela-internal-scripts", mountPath: "/vela/scripts"}, {name: "vela-internal-tools", mountPath: "/vela/tools"}] + }] + volumes: [ + {emptyDir: {} + name: "vela-internal-workspace" + }, + {emptyDir: {} + name: "vela-internal-home" + }, + {emptyDir: {} + name: "vela-internal-results" + }, + {emptyDir: {} + name: "vela-internal-steps" + }, + {emptyDir: {} + name: "vela-internal-scripts" + }, + {emptyDir: {} + name: "vela-internal-tools" + }, + {downwardAPI: { + defaultMode: 420 + items: [{ + fieldRef: { + apiVersion: "v1" + fieldPath: "metadata.annotations['vela.dev/ready']" + } + path: "ready" + }] + } + name: "vela-internal-downward" + }] + _volumes + restartPolicy: "Never" + } +} + +#StepContainer: { + _index: int + _waitFile: *"/vela/downward/ready" | string + if _index != 0 { + _waitFile: "/vela/tools/\(_index)" + } + _workspaceMounts: {...} + _volumeMounts: [ for v in _workspaceMounts {name: v.workspace.name, mountPath: v.mountPath}] + + name: string + args: ["-wait_file", _waitFile, "-wait_file_content", "-post_file", "/vela/tools/\(_index)", "-termination_path", "/vela/termination", "-step_metadata_dir", "/vela/steps/step-\(name)", "-step_metadata_dir_link", "/vela/steps/\(_index)", "-entrypoint", "/vela/scripts/script-\(_index)", "--"] + command: ["/vela/tools/entrypoint"] + env?: _ + image: string + imagePullPolicy: "Always" + terminationMessagePath: "/vela/termination" + terminationMessagePolicy: "File" + volumeMounts: [{ + name: "vela-internal-scripts" + mountPath: "/vela/scripts" + }, { + name: "vela-internal-tools" + mountPath: "/vela/tools" + }, { + name: "vela-internal-downward" + mountPath: "/vela/downward" + }, { + name: "vela-internal-workspace" + mountPath: "/workspace" + }, { + name: "vela-internal-home" + mountPath: "/vela/home" + }, { + name: "vela-internal-results" + mountPath: "/vela/results" + }, { + name: "vela-internal-steps" + mountPath: "/vela/steps" + }] + _volumeMounts +} diff --git a/pkg/utils/random.go b/pkg/utils/random.go new file mode 100644 index 000000000..99e50b9ff --- /dev/null +++ b/pkg/utils/random.go @@ -0,0 +1,34 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "math/rand" + "time" +) + +var letters = []rune("abcdefghijklmnopqrstuvwxyz123456789") + +// RandomString generate random string. +func RandomString(n int) string { + randSrc := rand.NewSource(time.Now().UnixNano()) + b := make([]rune, n) + for i := range b { + b[i] = letters[randSrc.Int63()%int64(len(letters))] + } + return string(b) +} diff --git a/pkg/utils/util_test.go b/pkg/utils/util_test.go new file mode 100644 index 000000000..aefc553ab --- /dev/null +++ b/pkg/utils/util_test.go @@ -0,0 +1,35 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "testing" +) + +func TestRandom(t *testing.T) { + s1 := RandomString(10) + s2 := RandomString(10) + if s1 == s2 { + t.Error("random generate same string") + } + if len(s1) != 10 { + t.Error("s1 length != 10") + } + if len(s2) != 10 { + t.Error("s2 length != 10") + } +}