Initial commit

This commit is contained in:
haochengkuo
2021-08-31 10:18:35 +08:00
parent 2ad6fb7b44
commit dc05a795b7
44 changed files with 5197 additions and 1 deletions

36
Dockerfile Normal file
View File

@@ -0,0 +1,36 @@
# Copyright 2021 Synology Inc.
############## Build stage ##############
FROM golang:1.13.6-alpine as builder
LABEL stage=synobuilder
RUN apk add --no-cache alpine-sdk
WORKDIR /go/src/synok8scsiplugin
COPY go.mod .
RUN go mod download
COPY Makefile .
COPY main.go .
COPY pkg ./pkg
RUN make
############## Final stage ##############
FROM alpine:latest
LABEL maintainers="Synology Authors" \
description="Synology CSI Plugin"
RUN apk add --no-cache e2fsprogs e2fsprogs-extra xfsprogs xfsprogs-extra blkid util-linux iproute2 bash
# Create symbolic link for chroot.sh
WORKDIR /
RUN mkdir /csibin
COPY chroot/chroot.sh /csibin
RUN chmod 777 /csibin/chroot.sh && ln -s /csibin/chroot.sh /csibin/iscsiadm
ENV PATH="/csibin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
# Copy and run CSI driver
COPY --from=builder /go/src/synok8scsiplugin/bin/synology-csi-driver synology-csi-driver
ENTRYPOINT ["/synology-csi-driver"]

202
LICENSE Normal file
View File

@@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

32
Makefile Normal file
View File

@@ -0,0 +1,32 @@
# Copyright 2021 Synology Inc.
REGISTRY_NAME=synology
IMAGE_NAME=synology-csi
IMAGE_VERSION=v1.0.0
IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_VERSION)
# For now, only build linux/amd64 platform
GOARCH?=amd64
BUILD_ENV=CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH)
BUILD_FLAGS="-extldflags \"-static\""
.PHONY: all clean synology-csi-driver synocli test docker-build
all: synology-csi-driver
synology-csi-driver:
@mkdir -p bin
$(BUILD_ENV) go build -v -ldflags $(BUILD_FLAGS) -o ./bin/synology-csi-driver ./
docker-build:
docker build -f Dockerfile -t $(IMAGE_TAG) .
synocli:
@mkdir -p bin
$(BUILD_ENV) go build -v -ldflags $(BUILD_FLAGS) -o ./bin/synocli ./synocli
test:
go test -v ./test/...
clean:
-rm -rf ./bin

190
README.md
View File

@@ -1 +1,189 @@
# synology-csi
# Synology CSI Driver for Kubernetes
The official [Container Storage Interface](https://github.com/container-storage-interface) driver for Synology NAS.
### Container Images & Kubernetes Compatibility
Driver Name: csi.san.synology.com
| Driver Version | Image | Supported K8s Version |
| -------------- | --------------------------------------------------------------------- | --------------------- |
| v1.0.0 | [synology-csi:v1.0.0](https://hub.docker.com/r/synology/synology-csi) | 1.19 |
The Synology CSI driver supports:
- Access Modes: Read/Write Multiple Pods
- Cloning
- Expansion
- Snapshot
## Installation
### Prerequisites
- Kubernetes versions 1.19
- Synology NAS running DSM 7.0 or above
- Go version 1.16 or above is recommended
- (Optional) Both [Volume Snapshot CRDs](https://github.com/kubernetes-csi/external-snapshotter/tree/v4.0.0/client/config/crd) and the [common snapshot controller](https://github.com/kubernetes-csi/external-snapshotter/tree/v4.0.0/deploy/kubernetes/snapshot-controller) must be installed in your Kubernetes cluster if you want to use the **Snapshot** feature
### Notice
1. Before installing the CSI driver, make sure you have created and initialized at least one **storage pool** and one **volume** on your DSM.
2. Make sure that all the worker nodes in your Kubernetes cluster can connect to your DSM.
3. After you complete the steps below, the *full* deployment of the CSI driver, including the snapshotter, will be installed. If you dont need the **Snapshot** feature, you can install the *basic* deployment of the CSI driver instead.
### Procedure
1. Clone the git repository. `git clone https://github.com/SynologyOpenSource/synology-csi.git`
2. Enter the directory. `cd synology-csi`
3. Copy the client-info-template.yml file. `cp config/client-info-template.yml config/client-info.yml`
4. Edit `config/client-info.yml` to configure the connection information for DSM. You can specify **one or more** storage systems on which the CSI volumes will be created. Change the following parameters as needed:
- *host*: The IPv4 address of your DSM.
- *port*: The port for connecting to DSM. The default HTTP port is 5000 and 5001 for HTTPS. Only change this if you use a different port.
- *https*: Set "true" to use HTTPS for secure connections. Make sure the port is properly configured as well.
- *username*, *password*: The credentials for connecting to DSM.
5. Run `./scripts/deploy.sh run` to install the driver. This will be a *full* deployment, which means you'll be building and running all CSI services as well as the snapshotter. If you want a *basic* deployment, which doesn't include installing a snapshotter, change the command as instructed below.
- *full*:
`./scripts/deploy.sh run`
- *basic*:
`./scripts/deploy.sh build && ./scripts/deploy.sh install --basic`
If you dont need to build the driver locally and want to pull the [image](https://hub.docker.com/r/synology/synology-csi) from Docker instead, run the command as instructed below.
- *full*:
`./scripts/deploy.sh install --all`
- *basic*:
`./scripts/deploy.sh install --basic`
Running the bash script will:
- Create a namespace named "`synology-csi`". This is where the driver will be installed.
- Create a secret named "`client-info-secret`" using the credentials from the client-info.yml you configured in the previous step.
- Build a local image and deploy the CSI driver.
- Create a **default** storage class named "`synology-iscsi-storage`" that uses the "`Retain`" policy.
- Create a volume snapshot class named "`synology-snapshotclass`" that uses the "`Delete`" policy. (*Full* deployment only)
6. Check if the status of all pods of the CSI driver is Running. `kubectl get pods -n synology-csi`
## CSI Driver Configuration
Storage classes and the secret are required for the CSI driver to function properly. This section explains how to do the following things:
1. Create the storage system secret (This is not mandatory because deploy.sh will complete all the configurations when you configure the config file mentioned previously.)
2. Configure storageclasses
3. Configure volumesnapshotclasses
### Creating a Secret
Create a secret to specify the storage system address and credentials (username and password). Usually the config file sets up the secret as well, but if you still want to create the secret or recreate it, follow the instructions below:
1. Edit the config file `config/client-info.yml` or create a new one like the example shown here:
```
clients:
- host: 192.168.1.1
port: 5000
https: false
username: <username>
password: <password>
- host: 192.168.1.2
port: 5001
https: true
username: <username>
password: <password>
```
The `clients` field can contain more than one Synology NAS. Seperate them with a prefix `-`.
2. Create the secret using the following command (usually done by deploy.sh):
```!
kubectl create secret -n <namespace> generic client-info-secret --from-file=config/client-info.yml
```
- Make sure to replace \<namespace\> with `synology-csi`. This is the default namespace. Change it to your custom namespace if needed.
- If you change the secret name "client-info-secret" to a different one, make sure that all files at `deploy/kubernetes/<k8s version>/` are using the secret name you set.
### Creating Storage Classes
Create and apply StorageClasses with the properties you want.
1. Create YAML files using the one at `deploy/kubernetes/<k8s version>/storage-class.yml` as the example, whose content is as below:
```
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
annotations:
storageclass.kubernetes.io/is-default-class: "false"
name: synostorage
provisioner: csi.san.synology.com
parameters:
fsType: 'ext4'
dsm: '192.168.1.1'
location: '/volume1'
reclaimPolicy: Retain
allowVolumeExpansion: true
```
2. Configure the StorageClass properties by assigning the parameters in the table. You can also leave blank if you dont have a preference:
| Name | Type | Description | Default |
| ---------- | ------ | ----------------------------------------------------------------------------------------------------------------- | ------- |
| *dsm* | string | The IPv4 address of your DSM, which must be included in the `client-info.yml` for the CSI driver to log in to DSM | - |
| *location* | string | The location (/volume1, /volume2, ...) on DSM where the LUN for *PersistentVolume* will be created | - |
| *fsType* | string | The formatting file system of the *PersistentVolumes* when you mount them on the pods | 'ext4' |
**Notice**
- If you leave the parameter *location* blank, the CSI driver will choose a volume on DSM with available storage to create the volumes.
- All volumes created by the CSI driver are Thin Provisioned LUNs on DSM. This will allow you to take snapshots of them.
3. Apply the YAML files to the Kubernetes cluster.
```
kubectl apply -f <storageclass_yaml>
```
### Creating Volume Snapshot Classes
Create and apply VolumeSnapshotClasses with the properties you want.
1. Create YAML files using the one at `deploy/kubernetes/<k8s version>/snapshotter/volume-snapshot-class.yml` as the example, whose content is as below:
```
apiVersion: snapshot.storage.k8s.io/v1beta1 # v1 for kubernetes v1.20 and above
kind: VolumeSnapshotClass
metadata:
name: synology-snapshotclass
annotations:
storageclass.kubernetes.io/is-default-class: "false"
driver: csi.san.synology.com
deletionPolicy: Delete
# parameters:
# description: 'Kubernetes CSI'
# is_locked: 'false'
```
2. Configure volume snapshot class properties by assigning the following parameters, all parameters are optional:
| Name | Type | Description | Default |
| ------------- | ------ | -------------------------------------------- | ------- |
| *description* | string | The description of the snapshot on DSM | "" |
| *is_locked* | string | Whether you want to lock the snapshot on DSM | 'false' |
3. Apply the YAML files to the Kubernetes cluster.
```
kubectl apply -f <volumesnapshotclass_yaml>
```
## Building & Manually Installing
By default, the CSI driver will pull the latest [image](https://hub.docker.com/r/synology/synology-csi) from Docker Hub.
If you want to use images you built locally for installation, edit all files under `deploy/kubernetes/<k8s version>/` and make sure `imagePullPolicy: IfNotPresent` is included in every csi-plugin container.
### Building
- To build the CSI driver, execute `make`.
- To build the *synocli* dev tool, execute `make synocli`. The output binary will be at `bin/synocli`.
- To run unit tests, execute `make test`.
- To build a docker image, run `./scripts/deploy.sh build`.
Afterwards, run `docker images` to check the newly created image.
### Installation
- To install all pods of the CSI driver, run `./scripts/deploy.sh install --all`
- To install pods of the CSI driver without the snapshotter, run `./scripts/deploy.sh install --basic`
- Run `./scripts/deploy.sh --help` to see more information on the usage of the commands.
### Uninstallation
If you are no longer using the CSI driver, make sure that no other resources in your Kubernetes cluster are using storage managed by Synology CSI driver before uninstalling it.
- `./scripts/uninstall.sh`

8
build.sh Executable file
View File

@@ -0,0 +1,8 @@
#!/bin/bash
SOURCE_PATH=$(realpath "$(dirname "${BASH_SOURCE}")")
echo "src path=""$SOURCE_PATH"
cd "$SOURCE_PATH" || exit
# Do docker build
make
make docker-build

12
chroot/chroot.sh Executable file
View File

@@ -0,0 +1,12 @@
#!/usr/bin/env bash
# This script is only used in the container, see Dockerfile.
DIR="/host" # csi-node mount / of the node to /host in the container
BIN="$(basename "$0")"
if [ -d "$DIR" ]; then
exec chroot $DIR /usr/bin/env -i PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" "$BIN" "$@"
fi
echo -n "Couldn't find hostPath: $DIR in the CSI container"
exit 1

View File

@@ -0,0 +1,18 @@
---
clients:
- host: 192.168.1.1
port: 5000
https: false
username: username
password: password
- host: 192.168.1.2
port: 5000
https: false
username: username
password: password
#host: # ipv4 address or domain of the DSM
#port: # port for connecting to the DSM
#https: # set this true to use https. you need to specify the port to DSM HTTPS port as well
#username: # username
#password: # password

View File

@@ -0,0 +1,162 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-controller-sa
namespace: synology-csi
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: synology-csi-controller-role
rules:
- apiGroups: [""]
resources: ["events"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update", "patch"]
- apiGroups: [""]
resources: ["persistentvolumeclaims/status"]
verbs: ["get", "list", "watch", "update", "patch"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["csinodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["csi.storage.k8s.io"]
resources: ["csinodeinfos"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update", "patch"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots"]
verbs: ["get", "list"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["get", "list"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: synology-csi-controller-role
namespace: synology-csi
subjects:
- kind: ServiceAccount
name: csi-controller-sa
namespace: synology-csi
roleRef:
kind: ClusterRole
name: synology-csi-controller-role
apiGroup: rbac.authorization.k8s.io
---
kind: StatefulSet
apiVersion: apps/v1
metadata:
name: synology-csi-controller
namespace: synology-csi
spec:
serviceName: "synology-csi-controller"
replicas: 1
selector:
matchLabels:
app: synology-csi-controller
template:
metadata:
labels:
app: synology-csi-controller
spec:
serviceAccountName: csi-controller-sa
hostNetwork: true
containers:
- name: csi-provisioner
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: quay.io/k8scsi/csi-provisioner:v1.6.0
args:
- --timeout=60s
- --csi-address=$(ADDRESS)
- --v=5
env:
- name: ADDRESS
value: /var/lib/csi/sockets/pluginproxy/csi.sock
imagePullPolicy: Always
volumeMounts:
- name: socket-dir
mountPath: /var/lib/csi/sockets/pluginproxy/
- name: csi-attacher
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: quay.io/k8scsi/csi-attacher:v2.1.0
args:
- --v=5
- --csi-address=$(ADDRESS)
env:
- name: ADDRESS
value: /var/lib/csi/sockets/pluginproxy/csi.sock
imagePullPolicy: Always
volumeMounts:
- name: socket-dir
mountPath: /var/lib/csi/sockets/pluginproxy/
- name: csi-resizer
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: quay.io/k8scsi/csi-resizer:v0.5.0
args:
- --v=5
- --csi-address=$(ADDRESS)
env:
- name: ADDRESS
value: /var/lib/csi/sockets/pluginproxy/csi.sock
imagePullPolicy: Always
volumeMounts:
- name: socket-dir
mountPath: /var/lib/csi/sockets/pluginproxy/
- name: csi-plugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: synology/synology-csi:v1.0.0
args:
- --nodeid=NotUsed
- --endpoint=$(CSI_ENDPOINT)
- --client-info
- /etc/synology/client-info.yml
- --log-level=info
env:
- name: CSI_ENDPOINT
value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
imagePullPolicy: IfNotPresent
volumeMounts:
- name: socket-dir
mountPath: /var/lib/csi/sockets/pluginproxy/
- name: client-info
mountPath: /etc/synology
readOnly: true
volumes:
- name: socket-dir
emptyDir: {}
- name: client-info
secret:
secretName: client-info-secret

View File

@@ -0,0 +1,9 @@
apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
name: csi.san.synology.com
spec:
attachRequired: true # Indicates the driver requires an attach operation (TODO: ControllerPublishVolume should be implemented)
podInfoOnMount: true
volumeLifecycleModes:
- Persistent

View File

@@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: synology-csi

View File

@@ -0,0 +1,139 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-node-sa
namespace: synology-csi
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: synology-csi-node-role
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "update"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: synology-csi-node-role
namespace: synology-csi
subjects:
- kind: ServiceAccount
name: csi-node-sa
namespace: synology-csi
roleRef:
kind: ClusterRole
name: synology-csi-node-role
apiGroup: rbac.authorization.k8s.io
---
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: synology-csi-node
namespace: synology-csi
spec:
selector:
matchLabels:
app: synology-csi-node
template:
metadata:
labels:
app: synology-csi-node
spec:
serviceAccount: csi-node-sa
hostNetwork: true
containers:
- name: csi-driver-registrar
securityContext:
privileged: true
imagePullPolicy: Always
image: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0
args:
- --v=5
- --csi-address=$(ADDRESS) # the csi socket path inside the pod
- --kubelet-registration-path=$(REGISTRATION_PATH) # the csi socket path on the host node
env:
- name: ADDRESS
value: /csi/csi.sock
- name: REGISTRATION_PATH
value: /var/lib/kubelet/plugins/csi.san.synology.com/csi.sock
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: plugin-dir
mountPath: /csi
- name: registration-dir
mountPath: /registration
- name: csi-plugin
securityContext:
privileged: true
imagePullPolicy: IfNotPresent
image: synology/synology-csi:v1.0.0
args:
- --nodeid=$(KUBE_NODE_NAME)
- --endpoint=$(CSI_ENDPOINT)
- --client-info
- /etc/synology/client-info.yml
- --log-level=info
env:
- name: CSI_ENDPOINT
value: unix://csi/csi.sock
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: kubelet-dir
mountPath: /var/lib/kubelet
mountPropagation: "Bidirectional"
- name: plugin-dir
mountPath: /csi
- name: client-info
mountPath: /etc/synology
readOnly: true
- name: host-root
mountPath: /host
- name: device-dir
mountPath: /dev
volumes:
- name: kubelet-dir
hostPath:
path: /var/lib/kubelet
type: Directory
- name: plugin-dir
hostPath:
path: /var/lib/kubelet/plugins/csi.san.synology.com/
type: DirectoryOrCreate
- name: registration-dir
hostPath:
path: /var/lib/kubelet/plugins_registry
type: Directory
- name: client-info
secret:
secretName: client-info-secret
- name: host-root
hostPath:
path: /
type: Directory
- name: device-dir
hostPath:
path: /dev
type: Directory

View File

@@ -0,0 +1,106 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-snapshotter-sa
namespace: synology-csi
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: synology-csi-snapshotter-role
rules:
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents/status"]
verbs: ["update"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: synology-csi-snapshotter-role
namespace: synology-csi
subjects:
- kind: ServiceAccount
name: csi-snapshotter-sa
namespace: synology-csi
roleRef:
kind: ClusterRole
name: synology-csi-snapshotter-role
apiGroup: rbac.authorization.k8s.io
---
kind: StatefulSet
apiVersion: apps/v1
metadata:
name: synology-csi-snapshotter
namespace: synology-csi
spec:
serviceName: "synology-csi-snapshotter"
replicas: 1
selector:
matchLabels:
app: synology-csi-snapshotter
template:
metadata:
labels:
app: synology-csi-snapshotter
spec:
serviceAccountName: csi-snapshotter-sa
hostNetwork: true
containers:
- name: csi-snapshotter
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: quay.io/k8scsi/csi-snapshotter:v3.0.3
args:
- --v=5
- --csi-address=$(ADDRESS)
env:
- name: ADDRESS
value: /var/lib/csi/sockets/pluginproxy/csi.sock
imagePullPolicy: Always
volumeMounts:
- name: socket-dir
mountPath: /var/lib/csi/sockets/pluginproxy/
- name: csi-plugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: synology/synology-csi:v1.0.0
args:
- --nodeid=NotUsed
- --endpoint=$(CSI_ENDPOINT)
- --client-info
- /etc/synology/client-info.yml
- --log-level=info
env:
- name: CSI_ENDPOINT
value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
imagePullPolicy: IfNotPresent
volumeMounts:
- name: socket-dir
mountPath: /var/lib/csi/sockets/pluginproxy/
- name: client-info
mountPath: /etc/synology
readOnly: true
volumes:
- name: socket-dir
emptyDir: {}
- name: client-info
secret:
secretName: client-info-secret

View File

@@ -0,0 +1,11 @@
apiVersion: snapshot.storage.k8s.io/v1beta1
kind: VolumeSnapshotClass
metadata:
name: synology-snapshotclass
annotations:
storageclass.kubernetes.io/is-default-class: "false"
driver: csi.san.synology.com
deletionPolicy: Delete
# parameters:
# description: 'Kubernetes CSI'
# is_locked: 'false'

View File

@@ -0,0 +1,14 @@
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: synology-iscsi-storage
# annotations:
# storageclass.kubernetes.io/is-default-class: "true"
provisioner: csi.san.synology.com
# if all params are empty, synology CSI will choose an available location to create volume
# parameters:
# dsm: '1.1.1.1'
# location: '/volume1'
# fsType: 'ext4'
reclaimPolicy: Retain
allowVolumeExpansion: true

21
go.mod Normal file
View File

@@ -0,0 +1,21 @@
module github.com/SynologyOpenSource/synology-csi
go 1.16
require (
github.com/antonfisher/nested-logrus-formatter v1.3.1
github.com/cenkalti/backoff/v4 v4.1.1
github.com/container-storage-interface/spec v1.3.0
github.com/golang/protobuf v1.4.3
github.com/kubernetes-csi/csi-lib-utils v0.9.1
github.com/kubernetes-csi/csi-test/v4 v4.2.0
github.com/sirupsen/logrus v1.7.0
github.com/spf13/cobra v1.1.3
github.com/spf13/pflag v1.0.5
google.golang.org/grpc v1.34.0
gopkg.in/yaml.v2 v2.4.0
k8s.io/mount-utils v0.21.2
k8s.io/utils v0.0.0-20210527160623-6fdb442a123b
)
exclude google.golang.org/grpc v1.37.0

545
go.sum Normal file
View File

@@ -0,0 +1,545 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630=
github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/antonfisher/nested-logrus-formatter v1.3.1 h1:NFJIr+pzwv5QLHTPyKz9UMEoHck02Q9L0FP13b/xSbQ=
github.com/antonfisher/nested-logrus-formatter v1.3.1/go.mod h1:6WTfyWFkBc9+zyBaKIqRrg/KwMqBbodBjgbHjDz7zjA=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ=
github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
github.com/container-storage-interface/spec v1.3.0 h1:wMH4UIoWnK/TXYw8mbcIHgZmB6kHOeIsYsiaTJwa6bc=
github.com/container-storage-interface/spec v1.3.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kubernetes-csi/csi-lib-utils v0.9.1 h1:sGq6ifVujfMSkfTsMZip44Ttv8SDXvsBlFk9GdYl/b8=
github.com/kubernetes-csi/csi-lib-utils v0.9.1/go.mod h1:8E2jVUX9j3QgspwHXa6LwyN7IHQDjW9jX3kwoWnSC+M=
github.com/kubernetes-csi/csi-test/v4 v4.2.0 h1:uyFJMSN9vnOOuQwndB43Kp4Bi/dScuATdv4FMuGJJQ8=
github.com/kubernetes-csi/csi-test/v4 v4.2.0/go.mod h1:HuWP7lCCJzehodzd4kO170soxqgzSQHZ5Jbp1pKPlmA=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.5 h1:obHEce3upls1IBn1gTw/o7bCv7OJb6Ib/o7wNO+4eKw=
github.com/nxadm/tail v1.4.5/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M=
github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.10.4 h1:NiTx7EEvBzu9sFOD1zORteLSt3o8gnlvZZwSE9TnY9U=
github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/robertkrimen/otto v0.0.0-20200922221731-ef014fd054ac/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 h1:lwlPPsmjDKK0J6eG6xDWd5XPehI0R024zxjDnw3esPA=
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d h1:MiWWjyhUzZ+jvhZvloX6ZrUsdEghn8a64Upd8EMHglE=
golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20201209185603-f92720507ed4 h1:J4dpx/41slnq1aogzUSTuBuvD7VXz7ZLkVpr32YgSlg=
google.golang.org/genproto v0.0.0-20201209185603-f92720507ed4/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
google.golang.org/grpc v1.34.0 h1:raiipEjMOIC/TO2AvyTxP25XFdLxNIBwzDh3FM3XztI=
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw=
k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=
k8s.io/client-go v0.19.0/go.mod h1:H9E/VT95blcFQnlyShFgnFT9ZnJOAceiUHM3MlRC+mU=
k8s.io/component-base v0.19.0/go.mod h1:dKsY8BxkA+9dZIAh2aWJLL/UdASFDNtGYTCItL4LM7Y=
k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o=
k8s.io/mount-utils v0.21.2 h1:tsCdw29Gv7PxwCQSeAmZ5WFkFLoNPdZVRouCnyDY/Lo=
k8s.io/mount-utils v0.21.2/go.mod h1:dwXbIPxKtTjrBEaX1aK/CMEf1KZ8GzMHpe3NEBfdFXI=
k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20210527160623-6fdb442a123b h1:MSqsVQ3pZvPGTqCjptfimO2WjG7A9un2zcpiHkA6M/s=
k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

107
main.go Normal file
View File

@@ -0,0 +1,107 @@
/*
* Copyright 2021 Synology Inc.
*/
package main
import (
"os"
"os/signal"
"syscall"
"github.com/spf13/cobra"
log "github.com/sirupsen/logrus"
"github.com/SynologyOpenSource/synology-csi/pkg/driver"
"github.com/SynologyOpenSource/synology-csi/pkg/dsm/common"
"github.com/SynologyOpenSource/synology-csi/pkg/dsm/service"
"github.com/SynologyOpenSource/synology-csi/pkg/logger"
)
var (
// CSI options
csiNodeID = "CSINode"
csiEndpoint = "unix:///var/lib/kubelet/plugins/" + driver.DriverName + "/csi.sock"
csiClientInfoPath = "/etc/synology/client-info.yml"
// Logging
logLevel = "info"
webapiDebug = false
)
var rootCmd = &cobra.Command{
Use: "synology-csi-driver",
Short: "Synology CSI Driver",
SilenceUsage: true,
RunE: func(cmd *cobra.Command, args []string) error {
if webapiDebug {
logger.WebapiDebug = true
logLevel = "debug"
}
logger.Init(logLevel)
err := driverStart()
if err != nil {
log.Errorf("Failed to driverStart(): %v", err)
return err
}
return nil
},
}
func driverStart() error {
log.Infof("CSI Options = {%s, %s, %s}", csiNodeID, csiEndpoint, csiClientInfoPath)
dsmService := service.NewDsmService()
// 1. Login DSMs by given ClientInfo
info, err := common.LoadConfig(csiClientInfoPath)
if err != nil {
log.Errorf("Failed to read config: %v", err)
return err
}
for _, client := range info.Clients {
err := dsmService.AddDsm(client)
if err != nil {
log.Errorf("Failed to add DSM: %s, error: %v", client.Host, err)
}
}
defer dsmService.RemoveAllDsms()
// 2. Create and Run the Driver
drv, err := driver.NewControllerAndNodeDriver(csiNodeID, csiEndpoint, dsmService)
if err != nil {
log.Errorf("Failed to create driver: %v", err)
return err
}
drv.Activate()
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
// Block until a signal is received.
<-c
log.Infof("Shutting down.")
return nil
}
func main() {
addFlags(rootCmd)
if err := rootCmd.Execute(); err != nil {
os.Exit(1)
}
os.Exit(0)
}
func addFlags(cmd *cobra.Command) {
cmd.PersistentFlags().StringVar(&csiNodeID, "nodeid", csiNodeID, "Node ID")
cmd.PersistentFlags().StringVarP(&csiEndpoint, "endpoint", "e", csiEndpoint, "CSI endpoint")
cmd.PersistentFlags().StringVarP(&csiClientInfoPath, "client-info", "f", csiClientInfoPath, "Path of Synology config yaml file")
cmd.PersistentFlags().StringVar(&logLevel, "log-level", logLevel, "Log level (debug, info, warn, error, fatal)")
cmd.PersistentFlags().BoolVarP(&webapiDebug, "debug", "d", webapiDebug, "Enable webapi debugging logs")
cmd.MarkFlagRequired("endpoint")
cmd.MarkFlagRequired("client-info")
cmd.Flags().SortFlags = false
cmd.PersistentFlags().SortFlags = false
}

View File

@@ -0,0 +1,493 @@
/*
Copyright 2021 Synology Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package driver
import (
"context"
"fmt"
"time"
"strconv"
"github.com/golang/protobuf/ptypes"
"github.com/container-storage-interface/spec/lib/go/csi"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/SynologyOpenSource/synology-csi/pkg/dsm/webapi"
"github.com/SynologyOpenSource/synology-csi/pkg/interfaces"
"github.com/SynologyOpenSource/synology-csi/pkg/models"
"github.com/SynologyOpenSource/synology-csi/pkg/utils"
)
type controllerServer struct {
Driver *Driver
dsmService interfaces.IDsmService
Initiator *initiatorDriver
}
func getSizeByCapacityRange(capRange *csi.CapacityRange) (int64, error) {
if capRange == nil {
return 1 * utils.UNIT_GB, nil
}
minSize := capRange.GetRequiredBytes()
maxSize := capRange.GetLimitBytes()
if 0 < maxSize && maxSize < minSize {
return 0, status.Error(codes.InvalidArgument, "Invalid input: limitBytes is smaller than requiredBytes")
}
if minSize < utils.UNIT_GB {
return 0, status.Error(codes.InvalidArgument, "Invalid input: required bytes is smaller than 1G")
}
return int64(minSize), nil
}
func (cs *controllerServer) isVolumeAccessModeSupport(mode csi.VolumeCapability_AccessMode_Mode) bool {
for _, accessMode := range cs.Driver.getVolumeCapabilityAccessModes() {
if mode == accessMode.Mode {
return true
}
}
return false
}
func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
sizeInByte, err := getSizeByCapacityRange(req.GetCapacityRange())
volName, volCap := req.GetName(), req.GetVolumeCapabilities()
volContentSrc := req.GetVolumeContentSource()
var srcSnapshotId string = ""
var srcVolumeId string = ""
var multiSession bool = false
if err != nil {
return nil, err
}
if volName == "" {
return nil, status.Errorf(codes.InvalidArgument, "No name is provided")
}
if volCap == nil {
return nil, status.Errorf(codes.InvalidArgument, "No volume capabilities are provided")
}
for _, cap := range volCap {
accessMode := cap.GetAccessMode().GetMode()
if !cs.isVolumeAccessModeSupport(accessMode) {
return nil, status.Errorf(codes.InvalidArgument, "Invalid volume capability access mode")
}
if accessMode == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER {
multiSession = false
} else if accessMode == csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER {
multiSession = true
}
}
if volContentSrc != nil {
if srcSnapshot := volContentSrc.GetSnapshot(); srcSnapshot != nil {
srcSnapshotId = srcSnapshot.SnapshotId
} else if srcVolume := volContentSrc.GetVolume(); srcVolume != nil {
srcVolumeId = srcVolume.VolumeId
} else {
return nil, status.Errorf(codes.InvalidArgument, "Invalid volume content source")
}
}
params := req.GetParameters()
isThin := true
if params["thin_provisioning"] != "" {
isThin = utils.StringToBoolean(params["thin_provisioning"])
}
spec := &models.CreateK8sVolumeSpec{
DsmIp: params["dsm"],
K8sVolumeName: volName,
LunName: fmt.Sprintf("%s-%s", models.LunPrefix, volName),
Location: params["location"],
Size: sizeInByte,
Type: params["type"],
ThinProvisioning: isThin,
TargetName: fmt.Sprintf("%s-%s", models.LunPrefix, volName),
MultipleSession: multiSession,
SourceSnapshotId: srcSnapshotId,
SourceVolumeId: srcVolumeId,
}
lunInfo, dsmIp, err := cs.dsmService.CreateVolume(spec)
if err != nil {
return nil, err
}
if int64(lunInfo.Size) != sizeInByte {
return nil , status.Errorf(codes.AlreadyExists, "Already existing volume name with different capacity")
}
return &csi.CreateVolumeResponse{
Volume: &csi.Volume{
VolumeId: lunInfo.Uuid,
CapacityBytes: int64(lunInfo.Size),
ContentSource: volContentSrc,
VolumeContext: map[string]string{
"dsm": dsmIp,
},
},
}, nil
}
func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
volumeId := req.GetVolumeId()
if volumeId == "" {
return nil, status.Errorf(codes.InvalidArgument, "No volume id is provided")
}
if err := cs.dsmService.DeleteVolume(volumeId); err != nil {
return nil, status.Errorf(codes.Internal,
fmt.Sprintf("Failed to DeleteVolume(%s), err: %v", volumeId, err))
}
return &csi.DeleteVolumeResponse{}, nil
}
func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) {
return nil, status.Error(codes.Unimplemented, "")
}
func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) {
return nil, status.Error(codes.Unimplemented, "")
}
func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) {
volumeId, volCap := req.GetVolumeId(), req.GetVolumeCapabilities()
if volumeId == "" {
return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request")
}
if volCap == nil {
return nil, status.Error(codes.InvalidArgument, "No volume capabilities are provided")
}
if cs.dsmService.GetVolume(volumeId) == nil {
return nil, status.Errorf(codes.NotFound, "Volume[%s] does not exist", volumeId)
}
for _, cap := range volCap {
if !cs.isVolumeAccessModeSupport(cap.GetAccessMode().GetMode()) {
return nil, status.Errorf(codes.NotFound, "Driver does not support volume capabilities:%v", volCap)
}
}
return &csi.ValidateVolumeCapabilitiesResponse{}, nil
}
func (cs *controllerServer) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) {
maxEntries := req.GetMaxEntries()
startingToken := req.GetStartingToken()
var entries []*csi.ListVolumesResponse_Entry
var nextToken string = ""
if 0 > maxEntries {
return nil, status.Error(codes.InvalidArgument, "Max entries can not be negative.")
}
pagingSkip := ("" != startingToken)
infos := cs.dsmService.ListVolumes()
var count int32 = 0
for _, info := range infos {
if info.Lun.Uuid == startingToken {
pagingSkip = false
}
if pagingSkip {
continue
}
if maxEntries > 0 && count >= maxEntries {
nextToken = info.Lun.Uuid
break
}
entries = append(entries, &csi.ListVolumesResponse_Entry{
Volume: &csi.Volume{
VolumeId: info.Lun.Uuid,
CapacityBytes: int64(info.Lun.Size),
VolumeContext: map[string]string{
"dsm": info.DsmIp,
"lunName": info.Lun.Name,
"targetIqn": info.Target.Iqn,
},
},
})
count++
}
if pagingSkip {
return nil, status.Errorf(codes.Aborted, fmt.Sprintf("Invalid StartingToken(%s)", startingToken))
}
return &csi.ListVolumesResponse{
Entries: entries,
NextToken: nextToken,
}, nil
}
func (cs *controllerServer) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) {
params := req.GetParameters()
volInfos, err := cs.dsmService.ListDsmVolumes(params["dsm"])
if err != nil {
return nil, status.Error(codes.InvalidArgument, "Failed to list dsm volumes")
}
var availableCapacity int64 = 0
location := params["location"]
for _, info := range volInfos {
if location != "" && info.Path != location {
continue
}
freeSize, err := strconv.ParseInt(info.Free, 10, 64)
if err != nil {
continue
}
availableCapacity += freeSize
}
return &csi.GetCapacityResponse{
AvailableCapacity: availableCapacity,
}, nil
}
func (cs *controllerServer) ControllerGetCapabilities(ctx context.Context, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) {
return &csi.ControllerGetCapabilitiesResponse{
Capabilities: cs.Driver.csCap,
}, nil
}
func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) {
srcVolId := req.GetSourceVolumeId()
snapshotName := req.GetName()
params := req.GetParameters()
if srcVolId == "" {
return nil, status.Error(codes.InvalidArgument, "Source volume id is empty.")
}
if snapshotName == "" {
return nil, status.Error(codes.InvalidArgument, "Snapshot name is empty.")
}
snapshotInfos, err := cs.dsmService.ListAllSnapshots()
if err != nil {
return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to ListAllSnapshots(), err: %v", err))
}
// idempotency
for _, snapshotInfo := range snapshotInfos {
if snapshotInfo.Name == snapshotName {
if snapshotInfo.ParentUuid != srcVolId {
return nil, status.Errorf(codes.AlreadyExists, fmt.Sprintf("Snapshot [%s] already exists but volume id is incompatible", snapshotName))
}
createTime, err := ptypes.TimestampProto(time.Unix(snapshotInfo.CreateTime, 0))
if err != nil {
return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to convert create time, err: %v", err))
}
return &csi.CreateSnapshotResponse{
Snapshot: &csi.Snapshot{
SizeBytes: snapshotInfo.TotalSize,
SnapshotId: snapshotInfo.Uuid,
SourceVolumeId: snapshotInfo.ParentUuid,
CreationTime: createTime,
ReadyToUse: (snapshotInfo.Status == "Healthy"),
},
}, nil
}
}
spec := &models.CreateK8sVolumeSnapshotSpec{
K8sVolumeId: srcVolId,
SnapshotName: snapshotName,
Description: params["description"],
TakenBy: models.K8sCsiName,
IsLocked: utils.StringToBoolean(params["is_locked"]),
}
snapshotId, err := cs.dsmService.CreateSnapshot(spec)
if err != nil {
if err == utils.OutOfFreeSpaceError("") || err == utils.SnapshotReachMaxCountError("") {
return nil,status.Errorf(codes.ResourceExhausted, fmt.Sprintf("Failed to CreateSnapshot(%s), err: %v", srcVolId, err))
} else {
return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to CreateSnapshot(%s), err: %v", srcVolId, err))
}
}
snapshotInfo, err := cs.dsmService.GetSnapshot(snapshotId)
if err != nil {
return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to GetSnapshot(%s), err: %v", snapshotId, err))
}
createTime, err := ptypes.TimestampProto(time.Unix(snapshotInfo.CreateTime, 0))
if err != nil {
return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to convert create time, err: %v", err))
}
return &csi.CreateSnapshotResponse{
Snapshot: &csi.Snapshot{
SizeBytes: snapshotInfo.TotalSize,
SnapshotId: snapshotInfo.Uuid,
SourceVolumeId: snapshotInfo.ParentUuid,
CreationTime: createTime,
ReadyToUse: (snapshotInfo.Status == "Healthy"),
},
}, nil
}
func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) {
snapshotId := req.GetSnapshotId()
if snapshotId == "" {
return nil, status.Error(codes.InvalidArgument, "Snapshot id is empty.")
}
err := cs.dsmService.DeleteSnapshot(snapshotId)
if err != nil {
return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to DeleteSnapshot(%s), err: %v", snapshotId, err))
}
return &csi.DeleteSnapshotResponse{}, nil
}
func (cs *controllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) {
srcVolId := req.GetSourceVolumeId()
snapshotId := req.GetSnapshotId()
maxEntries := req.GetMaxEntries()
startingToken := req.GetStartingToken()
var entries []*csi.ListSnapshotsResponse_Entry
var nextToken string = ""
if 0 > maxEntries {
return nil, status.Error(codes.InvalidArgument, "Max entries can not be negative.")
}
pagingSkip := ("" != startingToken)
var snapshotInfos []webapi.SnapshotInfo
var err error
if (srcVolId != "") {
snapshotInfos, err = cs.dsmService.ListSnapshots(srcVolId)
} else {
snapshotInfos, err = cs.dsmService.ListAllSnapshots()
}
if err != nil {
return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to ListSnapshots(%s), err: %v", srcVolId, err))
}
var count int32 = 0
for _, snapshotInfo := range snapshotInfos {
if snapshotInfo.Uuid == startingToken {
pagingSkip = false
}
if pagingSkip {
continue
}
if snapshotId != "" && snapshotInfo.Uuid != snapshotId {
continue
}
if maxEntries > 0 && count >= maxEntries {
nextToken = snapshotInfo.Uuid
break
}
createTime, err := ptypes.TimestampProto(time.Unix(snapshotInfo.CreateTime, 0))
if err != nil {
return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to convert create time, err: %v", err))
}
entries = append(entries, &csi.ListSnapshotsResponse_Entry{
Snapshot: &csi.Snapshot{
SizeBytes: snapshotInfo.TotalSize,
SnapshotId: snapshotInfo.Uuid,
SourceVolumeId: snapshotInfo.ParentUuid,
CreationTime: createTime,
ReadyToUse: (snapshotInfo.Status == "Healthy"),
},
})
count++
}
if pagingSkip {
return nil, status.Errorf(codes.Aborted, fmt.Sprintf("Invalid StartingToken(%s)", startingToken))
}
return &csi.ListSnapshotsResponse{
Entries: entries,
NextToken: nextToken,
}, nil
}
func (cs *controllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) {
volumeId, capRange := req.GetVolumeId(), req.GetCapacityRange()
if volumeId == "" || capRange == nil {
return nil, status.Error(codes.InvalidArgument,
"InvalidArgument: Please check volume ID and capacity range.")
}
sizeInByte, err := getSizeByCapacityRange(capRange)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument,
"InvalidArgument: Please check CapacityRange[%v]", capRange)
}
if err := cs.dsmService.ExpandLun(volumeId, sizeInByte); err != nil {
return nil, status.Error(codes.Internal,
fmt.Sprintf("Failed to expand volume [%s], err: %v", volumeId, err))
}
return &csi.ControllerExpandVolumeResponse{
CapacityBytes: sizeInByte,
NodeExpansionRequired: true,
}, nil
}
func (cs *controllerServer) ControllerGetVolume(ctx context.Context, req *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) {
return nil, status.Error(codes.Unimplemented, "")
}

129
pkg/driver/driver.go Normal file
View File

@@ -0,0 +1,129 @@
/*
Copyright 2021 Synology Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package driver
import (
"github.com/container-storage-interface/spec/lib/go/csi"
log "github.com/sirupsen/logrus"
"github.com/SynologyOpenSource/synology-csi/pkg/interfaces"
)
const (
DriverName = "csi.san.synology.com" // CSI dirver name
DriverVersion = "1.0.0"
)
type IDriver interface {
Activate()
}
type Driver struct {
// *csicommon.CSIDriver
name string
nodeID string
version string
endpoint string
csCap []*csi.ControllerServiceCapability
vCap []*csi.VolumeCapability_AccessMode
nsCap []*csi.NodeServiceCapability
DsmService interfaces.IDsmService
}
func NewControllerAndNodeDriver(nodeID string, endpoint string, dsmService interfaces.IDsmService) (*Driver, error) {
log.Debugf("NewControllerAndNodeDriver: DriverName: %v, DriverVersion: %v", DriverName, DriverVersion)
// TODO version format and validation
d := &Driver{
name: DriverName,
version: DriverVersion,
nodeID: nodeID,
endpoint: endpoint,
DsmService: dsmService,
}
d.addControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{
csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
csi.ControllerServiceCapability_RPC_LIST_VOLUMES,
csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
csi.ControllerServiceCapability_RPC_CLONE_VOLUME,
csi.ControllerServiceCapability_RPC_GET_CAPACITY,
})
d.addVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{
csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
})
d.addNodeServiceCapabilities([]csi.NodeServiceCapability_RPC_Type{
csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME,
csi.NodeServiceCapability_RPC_EXPAND_VOLUME,
})
log.Infof("New driver created: name=%s, nodeID=%s, version=%s, endpoint=%s", d.name, d.nodeID, d.version, d.endpoint)
return d, nil
}
// TODO: func NewNodeDriver() {}
// TODO: func NewControllerDriver() {}
func (d *Driver) Activate() {
go func() {
RunControllerandNodePublishServer(d.endpoint, d, NewControllerServer(d), NewNodeServer(d))
}()
}
func (d *Driver) addControllerServiceCapabilities(cl []csi.ControllerServiceCapability_RPC_Type) {
var csc []*csi.ControllerServiceCapability
for _, c := range cl {
log.Debugf("Enabling controller service capability: %v", c.String())
csc = append(csc, NewControllerServiceCapability(c))
}
d.csCap = csc
return
}
func (d *Driver) addVolumeCapabilityAccessModes(vc []csi.VolumeCapability_AccessMode_Mode) {
var vca []*csi.VolumeCapability_AccessMode
for _, c := range vc {
log.Debugf("Enabling volume access mode: %v", c.String())
vca = append(vca, NewVolumeCapabilityAccessMode(c))
}
d.vCap = vca
return
}
func (d *Driver) addNodeServiceCapabilities(nsc []csi.NodeServiceCapability_RPC_Type) {
var nca []*csi.NodeServiceCapability
for _, c := range nsc {
log.Debugf("Enabling node service capability: %v", c.String())
nca = append(nca, NewNodeServiceCapability(c))
}
d.nsCap = nca
return
}
func (d *Driver) getVolumeCapabilityAccessModes() []*csi.VolumeCapability_AccessMode { // for debugging
return d.vCap
}

114
pkg/driver/grpc.go Normal file
View File

@@ -0,0 +1,114 @@
/*
Copyright 2021 Synology Inc.
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package driver
import (
"net"
"os"
"sync"
log "github.com/sirupsen/logrus"
"google.golang.org/grpc"
"github.com/container-storage-interface/spec/lib/go/csi"
)
// Defines Non blocking GRPC server interfaces
type NonBlockingGRPCServer interface {
// Start services at the endpoint
Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer)
// Waits for the service to stop
Wait()
// Stops the service gracefully
Stop()
// Stops the service forcefully
ForceStop()
}
func NewNonBlockingGRPCServer() NonBlockingGRPCServer {
return &nonBlockingGRPCServer{}
}
// NonBlocking server
type nonBlockingGRPCServer struct {
wg sync.WaitGroup
server *grpc.Server
}
func (s *nonBlockingGRPCServer) Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) {
s.wg.Add(1)
go s.serve(endpoint, ids, cs, ns)
return
}
func (s *nonBlockingGRPCServer) Wait() {
s.wg.Wait()
}
func (s *nonBlockingGRPCServer) Stop() {
s.server.GracefulStop()
}
func (s *nonBlockingGRPCServer) ForceStop() {
s.server.Stop()
}
func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) {
proto, addr, err := ParseEndpoint(endpoint)
if err != nil {
log.Fatal(err.Error())
}
if proto == "unix" {
addr = "/" + addr
if err := os.Remove(addr); err != nil && !os.IsNotExist(err) {
log.Fatalf("Failed to remove %s, error: %s", addr, err.Error())
}
}
listener, err := net.Listen(proto, addr)
if err != nil {
log.Fatalf("Failed to listen: %v", err)
}
opts := []grpc.ServerOption{
grpc.UnaryInterceptor(logGRPC),
}
server := grpc.NewServer(opts...)
s.server = server
if ids != nil {
csi.RegisterIdentityServer(server, ids)
}
if cs != nil {
csi.RegisterControllerServer(server, cs)
}
if ns != nil {
csi.RegisterNodeServer(server, ns)
}
log.Infof("Listening for connections on address: %#v", listener.Addr())
if err := server.Serve(listener); err != nil {
log.Fatal(err.Error())
}
}

View File

@@ -0,0 +1,61 @@
/*
Copyright 2021 Synology Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package driver
import (
"context"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/container-storage-interface/spec/lib/go/csi"
)
type identityServer struct {
Driver *Driver
}
func (ids *identityServer) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) {
if ids.Driver.name == "" {
return nil, status.Error(codes.Unavailable, "Driver name not configured")
}
if ids.Driver.version == "" {
return nil, status.Error(codes.Unavailable, "Driver is missing version")
}
return &csi.GetPluginInfoResponse{
Name: ids.Driver.name,
VendorVersion: ids.Driver.version,
}, nil
}
func (ids *identityServer) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) {
return &csi.ProbeResponse{}, nil
}
func (ids *identityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) {
return &csi.GetPluginCapabilitiesResponse{
Capabilities: []*csi.PluginCapability{
{
Type: &csi.PluginCapability_Service_{
Service: &csi.PluginCapability_Service{
Type: csi.PluginCapability_Service_CONTROLLER_SERVICE,
},
},
},
},
}, nil
}

162
pkg/driver/initiator.go Normal file
View File

@@ -0,0 +1,162 @@
// Copyright 2021 Synology Inc.
package driver
import (
"errors"
"fmt"
"strings"
log "github.com/sirupsen/logrus"
utilexec "k8s.io/utils/exec"
)
type initiatorDriver struct {
chapUser string
chapPassword string
}
const (
ISCSIPort = 3260
)
func iscsiadm(cmdArgs ...string) utilexec.Cmd {
executor := utilexec.New()
return executor.Command("iscsiadm", cmdArgs...)
}
func iscsiadm_session() string {
cmd := iscsiadm("-m", "session")
out, err := cmd.CombinedOutput()
if err != nil {
exitErr, ok := err.(utilexec.ExitError)
if ok && exitErr.ExitStatus() == 21 { // iscsiadm: No active sessions
log.Info("No active iscsi session found.")
} else {
log.Errorf("Failed to run iscsiadm session: %v", err)
}
return ""
}
return string(out)
}
func iscsiadm_discovery(ip string) error {
cmd := iscsiadm(
"-m", "discoverydb",
"--type", "sendtargets",
"--portal", ip,
"--discover")
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("%s (%v)", string(out), err)
}
return nil
}
func iscsiadm_login(iqn, portal string) error {
cmd := iscsiadm(
"-m", "node",
"--targetname", iqn,
"--portal", portal,
"--login")
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("%s (%v)", string(out), err)
}
return nil
}
func iscsiadm_logout(iqn string) error {
cmd := iscsiadm(
"-m", "node",
"--targetname", iqn,
"--logout")
if _, err := cmd.CombinedOutput(); err != nil {
return err
}
return nil
}
func iscsiadm_rescan(iqn string) error {
cmd := iscsiadm(
"-m", "node",
"--targetname", iqn,
"-R")
if _, err := cmd.CombinedOutput(); err != nil {
return err
}
return nil
}
func hasSession(targetIqn string) bool{
sessions := iscsiadm_session();
for _, line := range strings.Split(sessions, "\n") {
if strings.Contains(line, targetIqn) {
return true
}
}
return false
}
func (d *initiatorDriver) login(targetIqn string, ip string) error{
portal := fmt.Sprintf("%s:%d", ip, ISCSIPort)
if (hasSession(targetIqn)) {
log.Infof("Session[%s] already exists.", targetIqn)
return nil
}
if err := iscsiadm_discovery(ip); err != nil {
log.Errorf("Failed in discovery of the target: %v", err)
return err
}
if err := iscsiadm_login(targetIqn, portal); err != nil {
log.Errorf("Failed in login of the target: %v", err)
return err
}
log.Infof("Login target portal [%s], iqn [%s].", portal, targetIqn)
return nil
}
func (d *initiatorDriver) logout(targetIqn string, ip string) error{
if (!hasSession(targetIqn)) {
log.Infof("Session[%s] doesn't exist.", targetIqn)
return nil
}
portal := fmt.Sprintf("%s:%d", ip, ISCSIPort)
if err := iscsiadm_logout(targetIqn); err != nil {
log.Errorf("Failed in logout of the target.\nTarget [%s], Portal [%s], Err[%v]",
targetIqn, portal, err)
return err
}
log.Infof("Logout target portal [%s], iqn [%s].", portal, targetIqn)
return nil
}
func (d *initiatorDriver) rescan(targetIqn string) error{
if (!hasSession(targetIqn)) {
msg := fmt.Sprintf("Session[%s] doesn't exist.", targetIqn)
log.Error(msg)
return errors.New(msg)
}
if err := iscsiadm_rescan(targetIqn); err != nil {
log.Errorf("Failed in rescan of the target.\nTarget [%s], Err[%v]",
targetIqn, err)
return err
}
log.Infof("Rescan target iqn [%s].", targetIqn)
return nil
}

403
pkg/driver/nodeserver.go Normal file
View File

@@ -0,0 +1,403 @@
/*
Copyright 2021 Synology Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package driver
import (
"context"
"fmt"
"os"
"strings"
"time"
"github.com/container-storage-interface/spec/lib/go/csi"
log "github.com/sirupsen/logrus"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"k8s.io/mount-utils"
"github.com/SynologyOpenSource/synology-csi/pkg/interfaces"
"github.com/SynologyOpenSource/synology-csi/pkg/utils"
)
type nodeServer struct {
Driver *Driver
Mounter *mount.SafeFormatAndMount
dsmService interfaces.IDsmService
Initiator *initiatorDriver
}
func getExistedDevicePath(paths []string) string {
ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop()
timer := time.NewTimer(60 * time.Second)
defer timer.Stop()
for {
select {
case <-ticker.C:
for _, path := range paths {
exists, err := mount.PathExists(path)
if err == nil && exists == true {
return path
} else {
log.Errorf("Can't find device path [%s], err: %v", path, err)
}
}
case <-timer.C:
return ""
}
}
}
func (ns *nodeServer) getVolumeMountPath(volumeId string) string {
paths := []string{}
k8sVolume := ns.dsmService.GetVolume(volumeId)
if k8sVolume == nil {
log.Errorf("Failed to get Volume id:%d.")
return ""
}
// Assume target and lun 1-1 mapping
mappingIndex := k8sVolume.Target.MappedLuns[0].MappingIndex
ips, err := utils.LookupIPv4(k8sVolume.DsmIp)
if err != nil {
log.Errorf("Failed to lookup ipv4 for host: %s", k8sVolume.DsmIp)
paths = append(paths, fmt.Sprintf("%sip-%s:3260-iscsi-%s-lun-%d", "/dev/disk/by-path/", k8sVolume.DsmIp, k8sVolume.Target.Iqn, mappingIndex))
} else {
for _, ipv4 := range ips {
paths = append(paths, fmt.Sprintf("%sip-%s:3260-iscsi-%s-lun-%d", "/dev/disk/by-path/", ipv4, k8sVolume.Target.Iqn, mappingIndex))
}
}
path := getExistedDevicePath(paths)
if path == "" {
log.Errorf("Volume mount path is not exist.")
return ""
}
return path
}
func createTargetMountPath(mounter mount.Interface, mountPath string, isBlock bool) (bool, error) {
notMount, err := mount.IsNotMountPoint(mounter, mountPath)
if err != nil {
if os.IsNotExist(err) {
if isBlock {
pathFile, err := os.OpenFile(mountPath, os.O_CREATE|os.O_RDWR, 0750)
if err != nil {
log.Errorf("Failed to create mountPath:%s with error: %v", mountPath, err)
return notMount, err
}
if err = pathFile.Close(); err != nil {
log.Errorf("Failed to close mountPath:%s with error: %v", mountPath, err)
return notMount, err
}
} else {
err = os.MkdirAll(mountPath, 0750)
if err != nil {
return notMount, err
}
}
notMount = true
} else {
return false, err
}
}
return notMount, nil
}
func (ns *nodeServer) loginTarget(volumeId string) error {
k8sVolume := ns.dsmService.GetVolume(volumeId);
if k8sVolume == nil {
return status.Error(codes.NotFound, fmt.Sprintf("Volume[%s] is not found", volumeId))
}
if err := ns.Initiator.login(k8sVolume.Target.Iqn, k8sVolume.DsmIp); err != nil {
return status.Errorf(codes.Internal,
fmt.Sprintf("Failed to login with target iqn [%s], err: %v", k8sVolume.Target.Iqn, err))
}
return nil
}
func (ns *nodeServer) logoutTarget(volumeId string) {
k8sVolume := ns.dsmService.GetVolume(volumeId)
if k8sVolume == nil {
return
}
ns.Initiator.logout(k8sVolume.Target.Iqn, k8sVolume.DsmIp)
}
func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
volumeId, stagingTargetPath, volumeCapability :=
req.GetVolumeId(), req.GetStagingTargetPath(), req.GetVolumeCapability()
if volumeId == "" || stagingTargetPath == "" || volumeCapability == nil {
return nil, status.Error(codes.InvalidArgument,
"InvalidArgument: Please check volume ID, staging target path and volume capability.")
}
// if block mode, skip mount
if volumeCapability.GetBlock() != nil {
return &csi.NodeStageVolumeResponse{}, nil
}
if err := ns.loginTarget(volumeId); err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
volumeMountPath := ns.getVolumeMountPath(volumeId)
if volumeMountPath == "" {
return nil, status.Error(codes.Internal, "Can't get volume mount path")
}
notMount, err := ns.Mounter.Interface.IsLikelyNotMountPoint(stagingTargetPath)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
if !notMount {
return &csi.NodeStageVolumeResponse{}, nil
}
fsType := volumeCapability.GetMount().GetFsType()
mountFlags := volumeCapability.GetMount().GetMountFlags()
options := append([]string{"rw"}, mountFlags...)
if err = ns.Mounter.FormatAndMount(volumeMountPath, stagingTargetPath, fsType, options); err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
return &csi.NodeStageVolumeResponse{}, nil
}
func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
if req.GetVolumeId() == "" { // Useless, just for sanity check
return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request")
}
stagingTargetPath := req.GetStagingTargetPath()
if stagingTargetPath == "" {
return nil, status.Error(codes.InvalidArgument, "Target path missing in request")
}
notMount, err := mount.IsNotMountPoint(ns.Mounter.Interface, stagingTargetPath)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
if !notMount {
err = ns.Mounter.Interface.Unmount(stagingTargetPath)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
}
return &csi.NodeUnstageVolumeResponse{}, nil
}
func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
volumeId, targetPath, stagingTargetPath := req.GetVolumeId(), req.GetTargetPath(), req.GetStagingTargetPath()
isBlock := req.GetVolumeCapability().GetBlock() != nil
if volumeId == "" || targetPath == "" || stagingTargetPath == "" {
return nil, status.Error(codes.InvalidArgument,
"InvalidArgument: Please check volume ID, target path and staging target path.")
}
notMount, err := createTargetMountPath(ns.Mounter.Interface, targetPath, isBlock)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
if !notMount {
return &csi.NodePublishVolumeResponse{}, nil
}
if err := ns.loginTarget(volumeId); err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
volumeMountPath := ns.getVolumeMountPath(volumeId)
if volumeMountPath == "" {
return nil, status.Error(codes.Internal, "Can't get volume mount path")
}
options := []string{"bind"}
if req.GetReadonly() {
options = append(options, "ro")
}
if isBlock {
err = ns.Mounter.Interface.Mount(volumeMountPath, targetPath, "", options)
} else {
fsType := req.GetVolumeCapability().GetMount().GetFsType()
err = ns.Mounter.Interface.Mount(stagingTargetPath, targetPath, fsType, options)
}
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
return &csi.NodePublishVolumeResponse{}, nil
}
func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
volumeId, targetPath := req.GetVolumeId(), req.GetTargetPath()
if volumeId == "" {
return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request")
}
if targetPath == "" {
return nil, status.Error(codes.InvalidArgument, "Target path missing in request")
}
if _, err := os.Stat(targetPath); err != nil {
if os.IsNotExist(err){
return &csi.NodeUnpublishVolumeResponse{}, nil
}
return nil, status.Errorf(codes.Internal, err.Error())
}
notMount, err := mount.IsNotMountPoint(ns.Mounter.Interface, targetPath)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
if notMount {
return &csi.NodeUnpublishVolumeResponse{}, nil
}
needToLogout := true
list, err := ns.Mounter.Interface.GetMountRefs(targetPath)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
for _, path := range list {
filePrefix := "/var/lib/kubelet/pods/"
blkPrefix := "/var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/publish/"
if strings.HasPrefix(path, filePrefix) || strings.HasPrefix(path, blkPrefix) {
needToLogout = false
break
}
}
if err := ns.Mounter.Interface.Unmount(targetPath); err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
if err := os.Remove(targetPath); err != nil {
return nil, status.Errorf(codes.Internal, "Failed to remove target path.")
}
if needToLogout {
ns.logoutTarget(volumeId)
}
return &csi.NodeUnpublishVolumeResponse{}, nil
}
func (ns *nodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) {
log.Debugf("Using default NodeGetInfo, ns.Driver.nodeID = [%s]", ns.Driver.nodeID)
return &csi.NodeGetInfoResponse{
NodeId: ns.Driver.nodeID,
}, nil
}
func (ns *nodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
return &csi.NodeGetCapabilitiesResponse{
Capabilities: ns.Driver.nsCap,
}, nil
}
func (ns *nodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) {
volumeId, volumePath := req.GetVolumeId(), req.GetVolumePath()
if volumeId == "" || volumePath == "" {
return nil, status.Error(codes.InvalidArgument, "Invalid Argument")
}
k8sVolume := ns.dsmService.GetVolume(volumeId)
if k8sVolume == nil {
return nil, status.Error(codes.NotFound,
fmt.Sprintf("Volume[%s] is not found", volumeId))
}
notMount, err := mount.IsNotMountPoint(ns.Mounter.Interface, volumePath)
if err != nil || notMount {
return nil, status.Error(codes.NotFound,
fmt.Sprintf("Volume[%s] does not exist on the %s", volumeId, volumePath))
}
lun := k8sVolume.Lun
return &csi.NodeGetVolumeStatsResponse{
Usage: []*csi.VolumeUsage{
&csi.VolumeUsage{
Available: int64(lun.Size - lun.Used),
Total: int64(lun.Size),
Used: int64(lun.Used),
Unit: csi.VolumeUsage_BYTES,
},
},
}, nil
}
func (ns *nodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) {
volumeId, volumePath := req.GetVolumeId(), req.GetVolumePath()
sizeInByte, err := getSizeByCapacityRange(req.GetCapacityRange())
if volumeId == "" || volumePath == "" {
return nil, status.Error(codes.InvalidArgument, "InvalidArgument: Please check volume ID and volume path.")
}
k8sVolume := ns.dsmService.GetVolume(volumeId)
if k8sVolume == nil {
return nil, status.Error(codes.NotFound, fmt.Sprintf("Volume[%s] is not found", volumeId))
}
if err := ns.Initiator.rescan(k8sVolume.Target.Iqn); err != nil {
return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to rescan. err: %v", err))
}
isBlock := req.GetVolumeCapability() != nil && req.GetVolumeCapability().GetBlock() != nil
if isBlock {
return &csi.NodeExpandVolumeResponse{
CapacityBytes: sizeInByte}, nil
}
volumeMountPath := ns.getVolumeMountPath(volumeId)
if volumeMountPath == "" {
return nil, status.Error(codes.Internal, "Can't get volume mount path")
}
ok, err := mount.NewResizeFs(ns.Mounter.Exec).Resize(volumeMountPath, volumePath)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
if !ok {
return nil, status.Error(codes.Internal, "Failed to expand volume filesystem")
}
return &csi.NodeExpandVolumeResponse{
CapacityBytes: sizeInByte}, nil
}

113
pkg/driver/utils.go Normal file
View File

@@ -0,0 +1,113 @@
/*
Copyright 2021 Synology Inc.
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package driver
import (
"context"
"fmt"
"strings"
"github.com/container-storage-interface/spec/lib/go/csi"
log "github.com/sirupsen/logrus"
"github.com/kubernetes-csi/csi-lib-utils/protosanitizer"
"google.golang.org/grpc"
"k8s.io/utils/exec"
"k8s.io/mount-utils"
)
func ParseEndpoint(ep string) (string, string, error) {
if strings.HasPrefix(strings.ToLower(ep), "unix://") || strings.HasPrefix(strings.ToLower(ep), "tcp://") {
s := strings.SplitN(ep, "://", 2)
if s[1] != "" {
return s[0], s[1], nil
}
}
return "", "", fmt.Errorf("Invalid endpoint: %v", ep)
}
func NewControllerServer(d *Driver) *controllerServer {
return &controllerServer{
Driver: d,
dsmService: d.DsmService,
}
}
func NewNodeServer(d *Driver) *nodeServer {
return &nodeServer{
Driver: d,
dsmService: d.DsmService,
Mounter: &mount.SafeFormatAndMount{
Interface: mount.New(""),
Exec: exec.New(),
},
Initiator: &initiatorDriver{
chapUser: "",
chapPassword: "",
},
}
}
func NewIdentityServer(d *Driver) *identityServer {
return &identityServer{
Driver: d,
}
}
func NewVolumeCapabilityAccessMode(mode csi.VolumeCapability_AccessMode_Mode) *csi.VolumeCapability_AccessMode {
return &csi.VolumeCapability_AccessMode{Mode: mode}
}
func NewControllerServiceCapability(cap csi.ControllerServiceCapability_RPC_Type) *csi.ControllerServiceCapability {
return &csi.ControllerServiceCapability{
Type: &csi.ControllerServiceCapability_Rpc{
Rpc: &csi.ControllerServiceCapability_RPC{
Type: cap,
},
},
}
}
func NewNodeServiceCapability(cap csi.NodeServiceCapability_RPC_Type) *csi.NodeServiceCapability {
return &csi.NodeServiceCapability{
Type: &csi.NodeServiceCapability_Rpc{
Rpc: &csi.NodeServiceCapability_RPC{
Type: cap,
},
},
}
}
func RunControllerandNodePublishServer(endpoint string, d *Driver, cs csi.ControllerServer, ns csi.NodeServer) {
ids := NewIdentityServer(d)
s := NewNonBlockingGRPCServer()
s.Start(endpoint, ids, cs, ns)
}
func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
log.Infof("GRPC call: %s", info.FullMethod)
log.Infof("GRPC request: %s", protosanitizer.StripSecrets(req))
resp, err := handler(ctx, req)
if err != nil {
log.Errorf("GRPC error: %v", err)
} else {
log.Infof("GRPC response: %s", protosanitizer.StripSecrets(resp))
}
return resp, err
}

40
pkg/dsm/common/config.go Normal file
View File

@@ -0,0 +1,40 @@
/*
* Copyright 2021 Synology Inc.
*/
package common
import (
"io/ioutil"
"gopkg.in/yaml.v2"
log "github.com/sirupsen/logrus"
)
type ClientInfo struct {
Host string `yaml:"host"`
Port int `yaml:"port"`
Https bool `yaml:"https"`
Username string `yaml:"username"`
Password string `yaml:"password"`
}
type SynoInfo struct {
Clients []ClientInfo `yaml:"clients"`
}
func LoadConfig(configPath string) (*SynoInfo, error) {
file, err := ioutil.ReadFile(configPath)
if err != nil {
log.Errorf("Unable to open config file: %v", err)
return nil, err
}
info := SynoInfo{}
err = yaml.Unmarshal(file, &info)
if err != nil {
log.Errorf("Failed to parse config: %v", err)
return nil, err
}
return &info, nil
}

624
pkg/dsm/service/dsm.go Normal file
View File

@@ -0,0 +1,624 @@
/*
* Copyright 2021 Synology Inc.
*/
package service
import (
"errors"
"fmt"
"github.com/cenkalti/backoff/v4"
log "github.com/sirupsen/logrus"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"strconv"
"time"
"strings"
"github.com/SynologyOpenSource/synology-csi/pkg/dsm/common"
"github.com/SynologyOpenSource/synology-csi/pkg/dsm/webapi"
"github.com/SynologyOpenSource/synology-csi/pkg/models"
"github.com/SynologyOpenSource/synology-csi/pkg/utils"
)
type DsmService struct {
dsms map[string]*webapi.DSM
}
func NewDsmService() *DsmService {
return &DsmService{
dsms: make(map[string]*webapi.DSM),
}
}
func (service *DsmService) AddDsm(client common.ClientInfo) error {
// TODO: use sn or other identifiers as key
if _, ok := service.dsms[client.Host]; ok {
log.Infof("Adding DSM [%s] already present.", client.Host)
return nil
}
dsm := &webapi.DSM{
Ip: client.Host,
Port: client.Port,
Username: client.Username,
Password: client.Password,
Https: client.Https,
}
err := dsm.Login()
if err != nil {
return fmt.Errorf("Failed to login to DSM: [%s]. err: %v", dsm.Ip, err)
}
service.dsms[dsm.Ip] = dsm
log.Infof("Add DSM [%s].", dsm.Ip)
return nil
}
func (service *DsmService) RemoveAllDsms() {
for _, dsm := range service.dsms {
log.Infof("Going to logout DSM [%s]", dsm.Ip)
for i := 0; i < 3; i++ {
err := dsm.Logout()
if err == nil {
break
}
log.Debugf("Retry to logout DSM [%s], retry: %d", dsm.Ip, i)
}
}
return
}
func (service *DsmService) GetDsm(ip string) (*webapi.DSM, error) {
dsm, ok := service.dsms[ip]
if !ok {
return nil, fmt.Errorf("Requested dsm [%s] does not exist", ip)
}
return dsm, nil
}
func (service *DsmService) GetDsmsCount() int {
return len(service.dsms)
}
func (service *DsmService) ListDsmVolumes(ip string) ([]webapi.VolInfo, error) {
var allVolInfos []webapi.VolInfo
for _, dsm := range service.dsms {
if ip != "" && dsm.Ip != ip {
continue
}
volInfos, err := dsm.VolumeList()
if err != nil {
continue
}
allVolInfos = append(allVolInfos, volInfos...)
}
return allVolInfos, nil
}
func (service *DsmService) getFirstAvailableVolume(dsm *webapi.DSM, sizeInBytes int64) (webapi.VolInfo, error) {
volInfos, err := dsm.VolumeList()
if err != nil {
return webapi.VolInfo{}, err
}
for _, volInfo := range volInfos {
free, err := strconv.ParseInt(volInfo.Free, 10, 64)
if err != nil {
return webapi.VolInfo{}, err
}
if free < utils.UNIT_GB {
continue
}
if volInfo.Status == "crashed" || volInfo.Status == "read_only" || volInfo.Status == "deleting" || free <= sizeInBytes {
continue
}
// ignore esata disk
if volInfo.Container == "external" && volInfo.Location == "sata" {
continue
}
return volInfo, nil
}
return webapi.VolInfo{}, fmt.Errorf("Cannot find any available volume")
}
func getLunTypeByInputParams(lunType string, isThin bool, locationFsType string) (string, error) {
log.Debugf("Input lunType: %v, isThin: %v, locationFsType: %v", lunType, isThin, locationFsType)
if lunType != "" {
return lunType, nil
}
if locationFsType == models.FsTypeExt4 {
if isThin {
return models.LunTypeAdv, nil // ADV
} else {
return models.LunTypeFile, nil // FILE
}
} else if locationFsType == models.FsTypeBtrfs {
if isThin {
return models.LunTypeBlun, nil // BLUN
} else {
return models.LunTypeBlunThick, nil // BLUN_THICK
}
}
return "", fmt.Errorf("Unknown volume fs type: %s", locationFsType)
}
func (service *DsmService) createMappingTarget(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec, lunUuid string) error {
dsmInfo, err := dsm.DsmInfoGet()
if err != nil {
return status.Errorf(codes.Internal, fmt.Sprintf("Failed to get DSM[%s] info", dsm.Ip));
}
genTargetIqn := func() string {
iqn := models.IqnPrefix + fmt.Sprintf("%s.%s", dsmInfo.Hostname, spec.K8sVolumeName)
iqn = strings.ReplaceAll(iqn, "_", "-")
iqn = strings.ReplaceAll(iqn, "+", "p")
if len(iqn) > models.MaxIqnLen {
return iqn[:models.MaxIqnLen]
}
return iqn
}
targetSpec := webapi.TargetCreateSpec{
Name: spec.TargetName,
Iqn: genTargetIqn(),
}
log.Debugf("TargetCreate spec: %v", targetSpec)
targetId, err := dsm.TargetCreate(targetSpec)
if err != nil && !errors.Is(err, utils.AlreadyExistError("")) {
return status.Errorf(codes.Internal, fmt.Sprintf("Failed to create target with spec: %v, err: %v", targetSpec, err))
}
if targetInfo, err := dsm.TargetGet(targetSpec.Name); err != nil {
return status.Errorf(codes.Internal, fmt.Sprintf("Failed to get target with spec: %v, err: %v", targetSpec, err))
} else {
targetId = strconv.Itoa(targetInfo.TargetId);
}
if spec.MultipleSession == true {
if err := dsm.TargetSet(targetId, 0); err != nil {
return status.Errorf(codes.Internal, fmt.Sprintf("Failed to set target [%s] max session, err: %v", spec.TargetName, err))
}
}
err = dsm.LunMapTarget([]string{targetId}, lunUuid)
if err != nil {
return status.Errorf(codes.Internal, fmt.Sprintf("Failed to map target [%s] to lun [%s], err: %v", spec.TargetName, lunUuid, err))
}
return nil
}
func (service *DsmService) createVolumeByDsm(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec) (webapi.LunInfo, error) {
// 1. Find a available location
if spec.Location == "" {
vol, err := service.getFirstAvailableVolume(dsm, spec.Size)
if err != nil {
return webapi.LunInfo{},
status.Errorf(codes.Internal, fmt.Sprintf("Failed to get available location, err: %v", err))
}
spec.Location = vol.Path
}
// 2. Check if location exists
dsmVolInfo, err := dsm.VolumeGet(spec.Location)
if err != nil {
return webapi.LunInfo{},
status.Errorf(codes.InvalidArgument, fmt.Sprintf("Unable to find location %s", spec.Location))
}
lunType, err := getLunTypeByInputParams(spec.Type, spec.ThinProvisioning, dsmVolInfo.FsType)
if err != nil {
return webapi.LunInfo{},
status.Errorf(codes.InvalidArgument, fmt.Sprintf("Unknown volume fs type: %s, location: %s", dsmVolInfo.FsType, spec.Location))
}
// 3. Create LUN
lunSpec := webapi.LunCreateSpec{
Name: spec.LunName,
Location: spec.Location,
Size: spec.Size,
Type: lunType,
}
log.Debugf("LunCreate spec: %v", lunSpec)
_, err = dsm.LunCreate(lunSpec)
if err != nil && !errors.Is(err, utils.AlreadyExistError("")) {
return webapi.LunInfo{},
status.Errorf(codes.Internal, fmt.Sprintf("Failed to create Volume with name: %s, err: %v", spec.K8sVolumeName, err))
}
// No matter lun existed or not, Get Lun by name
lunInfo, err := dsm.LunGet(spec.LunName)
if err != nil {
return webapi.LunInfo{},
// discussion with log
status.Errorf(codes.Internal, fmt.Sprintf("Failed to get existed LUN with name: %s, err: %v", spec.LunName, err))
}
// 4. Create Target and Map to Lun
err = service.createMappingTarget(dsm, spec, lunInfo.Uuid)
if err != nil {
// FIXME need to delete lun and target
return webapi.LunInfo{},
status.Errorf(codes.Internal, fmt.Sprintf("Failed to create and map target, err: %v", err))
}
log.Debugf("[%s] CreateVolume Successfully. VolumeId: %s", dsm.Ip, lunInfo.Uuid);
return lunInfo, nil
}
func waitCloneFinished(dsm *webapi.DSM, lunName string) error {
cloneBackoff := backoff.NewExponentialBackOff()
cloneBackoff.InitialInterval = 1 * time.Second
cloneBackoff.Multiplier = 2
cloneBackoff.RandomizationFactor = 0.1
cloneBackoff.MaxElapsedTime = 20 * time.Second
checkFinished := func() error {
lunInfo, err := dsm.LunGet(lunName)
if err != nil {
return backoff.Permanent(fmt.Errorf("Failed to get existed LUN with name: %s, err: %v", lunName, err))
}
if lunInfo.IsActionLocked != false {
return fmt.Errorf("Clone not yet completed. Lun: %s", lunName)
}
return nil
}
cloneNotify := func(err error, duration time.Duration) {
log.Infof("Lun is being locked for lun clone, waiting %3.2f seconds .....", float64(duration.Seconds()))
}
if err := backoff.RetryNotify(checkFinished, cloneBackoff, cloneNotify); err != nil {
log.Errorf("Could not finish clone after %3.2f seconds. err: %v", float64(cloneBackoff.MaxElapsedTime.Seconds()), err)
return err
}
log.Debugf("Clone successfully. Lun: %v", lunName)
return nil
}
func (service *DsmService) createVolumeBySnapshot(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec, snapshotInfo webapi.SnapshotInfo) (webapi.LunInfo, error) {
if spec.Size != 0 && spec.Size != snapshotInfo.TotalSize {
return webapi.LunInfo{}, status.Errorf(codes.OutOfRange, "Lun size [%d] is not equal to snapshot size [%d]", spec.Size, snapshotInfo.TotalSize)
}
snapshotCloneSpec := webapi.SnapshotCloneSpec{
Name: spec.LunName,
SrcLunUuid: snapshotInfo.ParentUuid,
SrcSnapshotUuid: snapshotInfo.Uuid,
}
if _, err := dsm.SnapshotClone(snapshotCloneSpec); err != nil && !errors.Is(err, utils.AlreadyExistError("")) {
return webapi.LunInfo{},
status.Errorf(codes.Internal, fmt.Sprintf("Failed to create volume with source snapshot ID: %s, err: %v", snapshotInfo.Uuid, err))
}
if err := waitCloneFinished(dsm, spec.LunName); err != nil {
return webapi.LunInfo{}, status.Errorf(codes.Internal, err.Error())
}
lunInfo, err := dsm.LunGet(spec.LunName)
if err != nil {
return webapi.LunInfo{},
status.Errorf(codes.Internal, fmt.Sprintf("Failed to get existed LUN with name: %s, err: %v", spec.LunName, err))
}
err = service.createMappingTarget(dsm, spec, lunInfo.Uuid)
if err != nil {
// FIXME need to delete lun and target
return webapi.LunInfo{},
status.Errorf(codes.Internal, fmt.Sprintf("Failed to create and map target, err: %v", err))
}
log.Debugf("[%s] createVolumeBySnapshot Successfully. VolumeId: %s", dsm.Ip, lunInfo.Uuid);
return lunInfo, nil
}
func (service *DsmService) createVolumeByVolume(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec, srcLunInfo webapi.LunInfo) (webapi.LunInfo, error) {
if spec.Size != 0 && spec.Size != int64(srcLunInfo.Size) {
return webapi.LunInfo{}, status.Errorf(codes.OutOfRange, "Lun size [%d] is not equal to src lun size [%d]", spec.Size, srcLunInfo.Size)
}
if spec.Location == "" {
spec.Location = srcLunInfo.Location
}
lunCloneSpec := webapi.LunCloneSpec{
Name: spec.LunName,
SrcLunUuid: srcLunInfo.Uuid,
Location: spec.Location,
}
if _, err := dsm.LunClone(lunCloneSpec); err != nil && !errors.Is(err, utils.AlreadyExistError("")) {
return webapi.LunInfo{},
status.Errorf(codes.Internal, fmt.Sprintf("Failed to create volume with source volume ID: %s, err: %v", srcLunInfo.Uuid, err))
}
if err := waitCloneFinished(dsm, spec.LunName); err != nil {
return webapi.LunInfo{}, status.Errorf(codes.Internal, err.Error())
}
lunInfo, err := dsm.LunGet(spec.LunName)
if err != nil {
return webapi.LunInfo{},
status.Errorf(codes.Internal, fmt.Sprintf("Failed to get existed LUN with name: %s, err: %v", spec.LunName, err))
}
err = service.createMappingTarget(dsm, spec, lunInfo.Uuid)
if err != nil {
// FIXME need to delete lun and target
return webapi.LunInfo{},
status.Errorf(codes.Internal, fmt.Sprintf("Failed to create and map target, err: %v", err))
}
log.Debugf("[%s] createVolumeByVolume Successfully. VolumeId: %s", dsm.Ip, lunInfo.Uuid);
return lunInfo, nil
}
func (service *DsmService) CreateVolume(spec *models.CreateK8sVolumeSpec) (webapi.LunInfo, string, error) {
if spec.SourceVolumeId != "" {
/* Create volume by exists volume (Clone) */
k8sVolume := service.GetVolume(spec.SourceVolumeId)
if k8sVolume == nil {
return webapi.LunInfo{}, "", status.Errorf(codes.NotFound, fmt.Sprintf("No such volume id: %s", spec.SourceVolumeId))
}
dsm, err := service.GetDsm(k8sVolume.DsmIp)
if err != nil {
return webapi.LunInfo{}, "", status.Errorf(codes.Internal, fmt.Sprintf("Failed to get DSM[%s]", k8sVolume.DsmIp))
}
lunInfo, err := service.createVolumeByVolume(dsm, spec, k8sVolume.Lun)
return lunInfo, dsm.Ip, err
} else if spec.SourceSnapshotId != "" {
/* Create volume by snapshot */
for _, dsm := range service.dsms {
snapshotInfo, err := dsm.SnapshotGet(spec.SourceSnapshotId)
if err != nil {
continue
}
lunInfo, err := service.createVolumeBySnapshot(dsm, spec, snapshotInfo)
return lunInfo, dsm.Ip, err
}
return webapi.LunInfo{}, "", status.Errorf(codes.NotFound, fmt.Sprintf("No such snapshot id: %s", spec.SourceSnapshotId))
} else if spec.DsmIp != "" {
/* Create volume by specific dsm ip */
dsm, err := service.GetDsm(spec.DsmIp)
if err != nil {
return webapi.LunInfo{}, "", status.Errorf(codes.Internal, fmt.Sprintf("%v", err))
}
lunInfo, err := service.createVolumeByDsm(dsm, spec)
return lunInfo, dsm.Ip, err
} else {
/* Find appropriate dsm to create volume */
for _, dsm := range service.dsms {
lunInfo, err := service.createVolumeByDsm(dsm, spec)
if err != nil {
log.Errorf("[%s] Failed to create Volume: %v", dsm.Ip, err)
continue
}
return lunInfo, dsm.Ip, nil
}
return webapi.LunInfo{}, "", status.Errorf(codes.Internal, fmt.Sprintf("Couldn't find any host available to create Volume"))
}
}
func (service *DsmService) DeleteVolume(volumeId string) error {
k8sVolume := service.GetVolume(volumeId)
if k8sVolume == nil {
log.Infof("Skip delete volume[%s] that is no exist", volumeId)
return nil
}
lun, target := k8sVolume.Lun, k8sVolume.Target
dsm, err := service.GetDsm(k8sVolume.DsmIp)
if err != nil {
return status.Errorf(codes.Internal, fmt.Sprintf("Failed to get DSM[%s]", k8sVolume.DsmIp))
}
if err := dsm.LunDelete(lun.Uuid); err != nil {
return err
}
if len(target.MappedLuns) != 1 {
log.Infof("Skip deletes target[%s] that was mapped with lun. DSM[%s]", target.Name, dsm.Ip)
return nil
}
if err := dsm.TargetDelete(strconv.Itoa(target.TargetId)); err != nil {
return err
}
return nil
}
func (service *DsmService) listVolumesByDsm(dsm *webapi.DSM, infos *[]*models.ListK8sVolumeRespSpec) {
targetInfos, err := dsm.TargetList()
if err != nil {
log.Errorf("[%s] Failed to list targets: %v", dsm.Ip, err)
}
for _, target := range targetInfos {
// TODO: use target.ConnectedSessions to filter targets
for _, mapping := range target.MappedLuns {
lun, err := dsm.LunGet(mapping.LunUuid)
if err != nil {
log.Errorf("[%s] Failed to get LUN(%s): %v", dsm.Ip, mapping.LunUuid, err)
}
if !strings.HasPrefix(lun.Name, models.LunPrefix) {
continue
}
*infos = append(*infos, &models.ListK8sVolumeRespSpec{
DsmIp: dsm.Ip,
Target: target,
Lun: lun,
})
}
}
return
}
func (service *DsmService) ListVolumes() []*models.ListK8sVolumeRespSpec {
var infos []*models.ListK8sVolumeRespSpec
for _, dsm := range service.dsms {
service.listVolumesByDsm(dsm, &infos)
}
return infos
}
func (service *DsmService) GetVolume(lunUuid string) *models.ListK8sVolumeRespSpec {
volumes := service.ListVolumes()
for _, volume := range volumes {
if volume.Lun.Uuid == lunUuid {
return volume
}
}
return nil
}
func (service *DsmService) ExpandLun(lunUuid string, newSize int64) error {
k8sVolume := service.GetVolume(lunUuid);
if k8sVolume == nil {
return status.Errorf(codes.InvalidArgument, fmt.Sprintf("Can't find volume[%s].", lunUuid))
}
if int64(k8sVolume.Lun.Size) > newSize {
return status.Errorf(codes.InvalidArgument,
fmt.Sprintf("Failed to expand volume[%s], because expand size[%d] bigger than before[%d].",
lunUuid, newSize, k8sVolume.Lun.Size))
}
spec := webapi.LunUpdateSpec{
Uuid: lunUuid,
NewSize: uint64(newSize),
}
dsm, err := service.GetDsm(k8sVolume.DsmIp)
if err != nil {
return status.Errorf(codes.Internal, fmt.Sprintf("Failed to get DSM[%s]", k8sVolume.DsmIp))
}
if err := dsm.LunUpdate(spec); err != nil {
return status.Errorf(codes.InvalidArgument,
fmt.Sprintf("Failed to expand volume[%s]. err: %v", lunUuid, err))
}
return nil
}
func (service *DsmService) CreateSnapshot(spec *models.CreateK8sVolumeSnapshotSpec) (string, error) {
k8sVolume := service.GetVolume(spec.K8sVolumeId);
if k8sVolume == nil {
return "", status.Errorf(codes.NotFound, fmt.Sprintf("Can't find volume[%s].", spec.K8sVolumeId))
}
snapshotSpec := webapi.SnapshotCreateSpec{
Name: spec.SnapshotName,
LunUuid: spec.K8sVolumeId,
Description: spec.Description,
TakenBy: spec.TakenBy,
IsLocked: spec.IsLocked,
}
dsm, err := service.GetDsm(k8sVolume.DsmIp)
if err != nil {
return "", status.Errorf(codes.InvalidArgument, fmt.Sprintf("Failed to get dsm: %v", err))
}
snapshotUuid, err := dsm.SnapshotCreate(snapshotSpec)
if err != nil {
return "", err
}
return snapshotUuid, nil
}
func (service *DsmService) DeleteSnapshot(snapshotUuid string) error {
for _, dsm := range service.dsms {
_, err := dsm.SnapshotGet(snapshotUuid)
if err != nil {
continue
}
err = dsm.SnapshotDelete(snapshotUuid)
if err != nil {
return status.Errorf(codes.Internal, fmt.Sprintf("Failed to delete snapshot [%s]. err: %v", snapshotUuid, err))
}
return nil
}
return nil
}
func (service *DsmService) ListAllSnapshots() ([]webapi.SnapshotInfo, error) {
var allInfos []webapi.SnapshotInfo
for _, dsm := range service.dsms {
infos, err := dsm.SnapshotList("")
if err != nil {
continue
}
allInfos = append(allInfos, infos...)
}
return allInfos, nil
}
func (service *DsmService) ListSnapshots(lunUuid string) ([]webapi.SnapshotInfo, error) {
k8sVolume := service.GetVolume(lunUuid);
if k8sVolume == nil {
return []webapi.SnapshotInfo{}, nil // return empty when the volume does not exist
}
dsm, err := service.GetDsm(k8sVolume.DsmIp)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, fmt.Sprintf("Failed to get dsm: %v", err))
}
infos, err := dsm.SnapshotList(lunUuid)
if err != nil {
return nil, status.Errorf(codes.Internal,
fmt.Sprintf("Failed to list snapshot on lun [%s]. err: %v", lunUuid, err))
}
return infos, nil
}
func (service *DsmService) GetSnapshot(snapshotUuid string) (webapi.SnapshotInfo, error) {
for _, dsm := range service.dsms {
info, err := dsm.SnapshotGet(snapshotUuid)
if err != nil {
continue
}
return info, nil
}
return webapi.SnapshotInfo{}, status.Errorf(codes.Internal, fmt.Sprintf("No such snapshot uuid [%s]", snapshotUuid))
}

208
pkg/dsm/webapi/dsmwebapi.go Normal file
View File

@@ -0,0 +1,208 @@
/*
* Copyright 2021 Synology Inc.
*/
package webapi
import (
"crypto/tls"
"encoding/json"
"fmt"
"io/ioutil"
log "github.com/sirupsen/logrus"
"net/http"
"net/url"
"regexp"
"github.com/SynologyOpenSource/synology-csi/pkg/logger"
)
type DSM struct {
Ip string
Port int
Username string
Password string
Sid string
Https bool
}
type errData struct {
Code int `json:"code"`
}
type dsmApiResp struct {
Success bool `json:"success"`
Err errData `json:"error"`
}
type Response struct {
StatusCode int
ErrorCode int
Success bool
Data interface{}
}
func (dsm *DSM) sendRequest(data string, apiTemplate interface{}, params url.Values, cgiPath string) (Response, error) {
resp, err := dsm.sendRequestWithoutConnectionCheck(data, apiTemplate, params, cgiPath)
if err != nil && resp.ErrorCode == 119 { // WEBAPI_ERR_SID_NOT_FOUND
// Re-login
if err := dsm.Login(); err != nil {
return Response{}, fmt.Errorf("Failed to re-login to DSM: [%s]. err: %v", dsm.Ip, err)
}
log.Info("Re-login succeeded.")
return dsm.sendRequestWithoutConnectionCheck(data, apiTemplate, params, cgiPath);
}
return resp, err
}
func (dsm *DSM) sendRequestWithoutConnectionCheck(data string, apiTemplate interface{}, params url.Values, cgiPath string) (Response, error) {
client := &http.Client{}
var req *http.Request
var err error
var cgiUrl string
// Ex: http://10.12.12.14:5000/webapi/auth.cgi
if dsm.Https {
// TODO: input CA certificate and fill in tls config
// Skip Verify when https
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client = &http.Client{Transport: tr}
cgiUrl = fmt.Sprintf("https://%s:%d/%s", dsm.Ip, dsm.Port, cgiPath)
} else {
cgiUrl = fmt.Sprintf("http://%s:%d/%s", dsm.Ip, dsm.Port, cgiPath)
}
baseUrl, err := url.Parse(cgiUrl)
if err != nil {
return Response{}, err
}
baseUrl.RawQuery = params.Encode()
if logger.WebapiDebug {
log.Debugln(baseUrl.RawQuery)
}
if data != "" {
req, err = http.NewRequest("POST", baseUrl.String(), nil)
} else {
req, err = http.NewRequest("GET", baseUrl.String(), nil)
}
if dsm.Sid != "" {
cookie := http.Cookie{Name: "id", Value: dsm.Sid}
req.AddCookie(&cookie)
}
resp, err := client.Do(req)
if err != nil {
return Response{}, err
}
defer resp.Body.Close()
// For debug print text body
var bodyText []byte
if logger.WebapiDebug {
bodyText, err = ioutil.ReadAll(resp.Body)
if err != nil {
return Response{}, err
}
s := string(bodyText)
log.Debugln(s)
}
if resp.StatusCode != 200 && resp.StatusCode != 302 {
return Response{}, fmt.Errorf("Bad response status code: %d", resp.StatusCode)
}
// Strip data json data from response
type envelop struct {
dsmApiResp
Data json.RawMessage `json:"data"`
}
e := envelop{}
var outResp Response
if logger.WebapiDebug {
if err := json.Unmarshal(bodyText, &e); err != nil {
return Response{}, err
}
} else {
decoder := json.NewDecoder(resp.Body)
if err := decoder.Decode(&e); err != nil {
return Response{}, err
}
}
outResp.Success = e.Success
outResp.ErrorCode = e.Err.Code
outResp.StatusCode = resp.StatusCode
if !e.Success {
return outResp, fmt.Errorf("DSM Api error. Error code:%d", outResp.ErrorCode)
}
if e.Data != nil {
if err := json.Unmarshal(e.Data, apiTemplate); err != nil {
return Response{}, err
}
}
outResp.Data = apiTemplate
if err != nil {
return Response{}, err
}
return outResp, nil
}
// Login by given user name and password
func (dsm *DSM) Login() error {
params := url.Values{}
params.Add("api", "SYNO.API.Auth")
params.Add("method", "login")
params.Add("version", "3")
params.Add("account", dsm.Username)
params.Add("passwd", dsm.Password)
params.Add("format", "sid")
type LoginResp struct {
Sid string `json:"sid"`
}
resp, err := dsm.sendRequestWithoutConnectionCheck("", &LoginResp{}, params, "webapi/auth.cgi")
if err != nil {
r, _ := regexp.Compile("passwd=.*&")
temp := r.ReplaceAllString(err.Error(), "")
return fmt.Errorf("%s", temp)
}
loginResp, ok := resp.Data.(*LoginResp)
if !ok {
return fmt.Errorf("Failed to assert response to %T", &LoginResp{})
}
dsm.Sid = loginResp.Sid
return nil
}
// Logout on current IP and reset the synoToken
func (dsm *DSM) Logout() error {
params := url.Values{}
params.Add("api", "SYNO.API.Auth")
params.Add("method", "logout")
params.Add("version", "1")
_, err := dsm.sendRequestWithoutConnectionCheck("", &struct{}{}, params, "webapi/entry.cgi")
if err != nil {
return err
}
dsm.Sid = ""
return nil
}

500
pkg/dsm/webapi/iscsi.go Normal file
View File

@@ -0,0 +1,500 @@
// Copyright 2021 Synology Inc.
package webapi
import (
"encoding/json"
"fmt"
"net/url"
"strconv"
"strings"
log "github.com/sirupsen/logrus"
"github.com/SynologyOpenSource/synology-csi/pkg/logger"
"github.com/SynologyOpenSource/synology-csi/pkg/utils"
)
type LunInfo struct {
Name string `json:"name"`
Uuid string `json:"uuid"`
LunType int `json:"type"`
Location string `json:"location"`
Size uint64 `json:"size"`
Used uint64 `json:"allocated_size"`
Status string `json:"status"`
FlashcacheStatus string `json:"flashcache_status"`
IsActionLocked bool `json:"is_action_locked"`
}
type MappedLun struct {
LunUuid string `json:"lun_uuid"`
MappingIndex int `json:"mapping_index"`
}
type ConncetedSession struct {
Iqn string `json:"iqn"`
Ip string `json:"ip"`
}
type NetworkPortal struct {
ControllerId int `json:"controller_id"`
InterfaceName string `json:"interface_name"`
}
type TargetInfo struct {
Name string `json:"name"`
Iqn string `json:"iqn"`
Status string `json:"status"`
MaxSessions int `json:"max_sessions"`
MappedLuns []MappedLun `json:"mapped_luns"`
ConnectedSessions []ConncetedSession `json:"connected_sessions"`
NetworkPortals []NetworkPortal `json:"network_portals"`
TargetId int `json:"target_id"`
}
type SnapshotInfo struct {
Name string `json:"name"`
Uuid string `json:"uuid"`
ParentUuid string `json:"parent_uuid"`
Status string `json:"status"`
TotalSize int64 `json:"total_size"`
CreateTime int64 `json:"create_time"`
}
type LunDevAttrib struct {
DevAttrib string `json:"dev_attrib"`
Enable int `json:"enable"`
}
type LunCreateSpec struct {
Name string
Location string
Size int64
Type string
DevAttribs []LunDevAttrib
}
type LunUpdateSpec struct {
Uuid string
NewSize uint64
}
type LunCloneSpec struct {
Name string
SrcLunUuid string
Location string
}
type TargetCreateSpec struct {
Name string
Iqn string
}
type SnapshotCreateSpec struct {
Name string
LunUuid string
Description string
TakenBy string
IsLocked bool
}
type SnapshotCloneSpec struct {
Name string
SrcLunUuid string
SrcSnapshotUuid string
}
func errCodeMapping(errCode int, oriErr error) error {
switch errCode {
case 18990002: // Out of free space
return utils.OutOfFreeSpaceError("")
case 18990538: // Duplicated LUN name
return utils.AlreadyExistError("")
case 18990541:
return utils.LunReachMaxCountError("")
case 18990542:
return utils.TargetReachMaxCountError("")
case 18990744: // Duplicated Target name
return utils.AlreadyExistError("")
case 18990532:
return utils.NoSuchSnapshotError("")
case 18990500:
return utils.BadLunTypeError("")
case 18990543:
return utils.SnapshotReachMaxCountError("")
}
if errCode > 18990000 {
return utils.IscsiDefaultError("")
}
return oriErr
}
func (dsm *DSM) LunList() ([]LunInfo, error) {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.LUN")
params.Add("method", "list")
params.Add("version", "1")
params.Add("types", "[\"BLOCK\", \"FILE\", \"THIN\", \"ADV\", \"SINK\", \"CINDER\", \"CINDER_BLUN\", \"CINDER_BLUN_THICK\", \"BLUN\", \"BLUN_THICK\", \"BLUN_SINK\", \"BLUN_THICK_SINK\"]")
params.Add("additional", "[\"allocated_size\",\"status\",\"flashcache_status\", \"is_action_locked\"]")
type LunInfos struct {
Luns []LunInfo `json:"luns"`
}
resp, err := dsm.sendRequest("", &LunInfos{}, params, "webapi/entry.cgi")
if err != nil {
return nil, err
}
lunInfos, ok := resp.Data.(*LunInfos)
if !ok {
return nil, fmt.Errorf("Failed to assert response to %T", &LunInfos{})
}
return lunInfos.Luns, nil
}
func (dsm *DSM) LunCreate(spec LunCreateSpec) (string, error) {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.LUN")
params.Add("method", "create")
params.Add("version", "1")
params.Add("name", strconv.Quote(spec.Name))
params.Add("size", strconv.FormatInt(int64(spec.Size), 10))
params.Add("type", spec.Type)
params.Add("location", spec.Location)
js, err := json.Marshal(spec.DevAttribs)
if err != nil {
return "", err
}
params.Add("dev_attribs", string(js))
type LunCreateResp struct {
Uuid string `json:"uuid"`
}
resp, err := dsm.sendRequest("", &LunCreateResp{}, params, "webapi/entry.cgi")
err = errCodeMapping(resp.ErrorCode, err)
if err != nil {
return "", err
}
lunResp, ok := resp.Data.(*LunCreateResp)
if !ok {
return "", fmt.Errorf("Failed to assert response to %T", &LunCreateResp{})
}
return lunResp.Uuid, nil
}
func (dsm *DSM) LunUpdate(spec LunUpdateSpec) error {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.LUN")
params.Add("method", "set")
params.Add("version", "1")
params.Add("uuid", strconv.Quote(spec.Uuid))
params.Add("new_size", strconv.FormatInt(int64(spec.NewSize), 10))
_, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
if err != nil {
return err
}
return nil
}
func (dsm *DSM) LunGet(uuid string) (LunInfo, error) {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.LUN")
params.Add("method", "get")
params.Add("version", "1")
params.Add("uuid", strconv.Quote(uuid))
params.Add("additional", "[\"allocated_size\",\"status\",\"flashcache_status\", \"is_action_locked\"]")
type Info struct {
Lun LunInfo `json:"lun"`
}
info := Info{}
_, err := dsm.sendRequest("", &info, params, "webapi/entry.cgi")
if err != nil {
return LunInfo{}, err
}
return info.Lun, nil
}
func (dsm *DSM) LunClone(spec LunCloneSpec) (string, error) {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.LUN")
params.Add("method", "clone")
params.Add("version", "1")
params.Add("src_lun_uuid", strconv.Quote(spec.SrcLunUuid))
params.Add("dst_lun_name", strconv.Quote(spec.Name))
params.Add("dst_location", strconv.Quote(spec.Location))
type LunCloneResp struct {
Uuid string `json:"dst_lun_uuid"`
}
resp, err := dsm.sendRequest("", &LunCloneResp{}, params, "webapi/entry.cgi")
err = errCodeMapping(resp.ErrorCode, err)
if err != nil {
return "", err
}
cloneLunResp, ok := resp.Data.(*LunCloneResp)
if !ok {
return "", fmt.Errorf("Failed to assert response to %T", &LunCloneResp{})
}
return cloneLunResp.Uuid, nil
}
func (dsm *DSM) TargetList() ([]TargetInfo, error) {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.Target")
params.Add("method", "list")
params.Add("version", "1")
params.Add("additional", "[\"mapped_lun\", \"connected_sessions\"]")
type TargetInfos struct {
Targets []TargetInfo `json:"targets"`
}
resp, err := dsm.sendRequest("", &TargetInfos{}, params, "webapi/entry.cgi")
if err != nil {
return nil, err
}
trgInfos, ok := resp.Data.(*TargetInfos)
if !ok {
return nil, fmt.Errorf("Failed to assert response to %T", &TargetInfos{})
}
return trgInfos.Targets, nil
}
func (dsm *DSM) TargetGet(targetId string) (TargetInfo, error) {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.Target")
params.Add("method", "get")
params.Add("version", "1")
params.Add("target_id", strconv.Quote(targetId))
params.Add("additional", "[\"mapped_lun\", \"connected_sessions\"]")
type Info struct {
Target TargetInfo `json:"target"`
}
info := Info{}
_, err := dsm.sendRequest("", &info, params, "webapi/entry.cgi")
if err != nil {
return TargetInfo{}, err
}
return info.Target, nil
}
// Enable muti session
func (dsm *DSM) TargetSet(targetId string, maxSession int) error {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.Target")
params.Add("method", "set")
params.Add("version", "1")
params.Add("target_id", strconv.Quote(targetId))
params.Add("max_sessions", strconv.Itoa(maxSession))
_, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
if err != nil {
return err
}
return nil
}
func (dsm *DSM) TargetCreate(spec TargetCreateSpec) (string, error) {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.Target")
params.Add("method", "create")
params.Add("version", "1")
params.Add("name", spec.Name)
params.Add("auth_type", "0")
params.Add("iqn", spec.Iqn)
type TrgCreateResp struct {
TargetId int `json:"target_id"`
}
resp, err := dsm.sendRequest("", &TrgCreateResp{}, params, "webapi/entry.cgi")
err = errCodeMapping(resp.ErrorCode, err)
if err != nil {
return "", err
}
trgResp, ok := resp.Data.(*TrgCreateResp)
if !ok {
return "", fmt.Errorf("Failed to assert response to %T", &TrgCreateResp{})
}
return strconv.Itoa(trgResp.TargetId), nil
}
func (dsm *DSM) LunMapTarget(targetIds []string, lunUuid string) error {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.LUN")
params.Add("method", "map_target")
params.Add("version", "1")
params.Add("uuid", strconv.Quote(lunUuid))
params.Add("target_ids", fmt.Sprintf("[%s]", strings.Join(targetIds, ",")))
if logger.WebapiDebug {
log.Debugln(params)
}
_, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
if err != nil {
return err
}
return nil
}
func (dsm *DSM) LunDelete(lunUuid string) error {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.LUN")
params.Add("method", "delete")
params.Add("version", "1")
params.Add("uuid", strconv.Quote(lunUuid))
_, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
if err != nil {
return err
}
return nil
}
func (dsm *DSM) TargetDelete(targetName string) error {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.Target")
params.Add("method", "delete")
params.Add("version", "1")
params.Add("target_id", strconv.Quote(targetName))
_, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
if err != nil {
return err
}
return nil
}
func (dsm *DSM) SnapshotCreate(spec SnapshotCreateSpec) (string, error) {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.LUN")
params.Add("method", "take_snapshot")
params.Add("version", "1")
params.Add("src_lun_uuid", strconv.Quote(spec.LunUuid))
params.Add("snapshot_name", strconv.Quote(spec.Name))
params.Add("description", strconv.Quote(spec.Description))
params.Add("taken_by", strconv.Quote(spec.TakenBy))
params.Add("is_locked", strconv.FormatBool(spec.IsLocked))
params.Add("is_app_consistent", strconv.FormatBool(false))
type SnapshotCreateResp struct {
Uuid string `json:"snapshot_uuid"`
}
resp, err := dsm.sendRequest("", &SnapshotCreateResp{}, params, "webapi/entry.cgi")
if err != nil {
return "", errCodeMapping(resp.ErrorCode, err)
}
snapshotResp, ok := resp.Data.(*SnapshotCreateResp)
if !ok {
return "", fmt.Errorf("Failed to assert response to %T", &SnapshotCreateResp{})
}
return snapshotResp.Uuid, nil
}
func (dsm *DSM) SnapshotDelete(snapshotUuid string) error {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.LUN")
params.Add("method", "delete_snapshot")
params.Add("version", "1")
params.Add("snapshot_uuid", strconv.Quote(snapshotUuid))
resp, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
if err != nil {
return errCodeMapping(resp.ErrorCode, err)
}
return nil
}
func (dsm *DSM) SnapshotGet(snapshotUuid string) (SnapshotInfo, error) {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.LUN")
params.Add("method", "get_snapshot")
params.Add("version", "1")
params.Add("snapshot_uuid", strconv.Quote(snapshotUuid))
type Info struct {
Snapshot SnapshotInfo `json:"snapshot"`
}
info := Info{}
resp, err := dsm.sendRequest("", &info, params, "webapi/entry.cgi")
if err != nil {
return SnapshotInfo{}, errCodeMapping(resp.ErrorCode, err)
}
return info.Snapshot, nil
}
func (dsm *DSM) SnapshotList(lunUuid string) ([]SnapshotInfo, error) {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.LUN")
params.Add("method", "list_snapshot")
params.Add("version", "1")
params.Add("src_lun_uuid", strconv.Quote(lunUuid))
type Infos struct {
Snapshots []SnapshotInfo `json:"snapshots"`
}
resp, err := dsm.sendRequest("", &Infos{}, params, "webapi/entry.cgi")
if err != nil {
return nil, errCodeMapping(resp.ErrorCode, err)
}
infos, ok := resp.Data.(*Infos)
if !ok {
return nil, fmt.Errorf("Failed to assert response to %T", &Infos{})
}
return infos.Snapshots, nil
}
func (dsm *DSM) SnapshotClone(spec SnapshotCloneSpec) (string, error) {
params := url.Values{}
params.Add("api", "SYNO.Core.ISCSI.LUN")
params.Add("method", "clone_snapshot")
params.Add("version", "1")
params.Add("src_lun_uuid", strconv.Quote(spec.SrcLunUuid))
params.Add("snapshot_uuid", strconv.Quote(spec.SrcSnapshotUuid))
params.Add("cloned_lun_name", strconv.Quote(spec.Name))
type SnapshotCloneResp struct {
Uuid string `json:"cloned_lun_uuid"`
}
resp, err := dsm.sendRequest("", &SnapshotCloneResp{}, params, "webapi/entry.cgi")
if err != nil {
return "", errCodeMapping(resp.ErrorCode, err)
}
snapshotCloneResp, ok := resp.Data.(*SnapshotCloneResp)
if !ok {
return "", fmt.Errorf("Failed to assert response to %T", &SnapshotCloneResp{})
}
return snapshotCloneResp.Uuid, nil
}

87
pkg/dsm/webapi/share.go Normal file
View File

@@ -0,0 +1,87 @@
// Copyright 2021 Synology Inc.
package webapi
import (
"encoding/json"
"fmt"
"net/url"
"strconv"
)
type ShareInfo struct {
Name string `json:"name"`
VolPath string `json:"vol_path"`
Desc string `json:"desc"`
EnableShareCow bool `json:"enable_share_cow"`
EnableRecycleBin bool `json:"enable_recycle_bin"`
RecycleBinAdminOnly bool `json:"recycle_bin_admin_only"`
Encryption int `json:"encryption"`
}
type ShareCreateSpec struct {
Name string `json:"name"`
ShareInfo ShareInfo `json:"shareinfo"`
}
func (dsm *DSM) ShareList() ([]ShareInfo, error) {
params := url.Values{}
params.Add("api", "SYNO.Core.Share")
params.Add("method", "list")
params.Add("version", "1")
params.Add("additional", "[\"encryption\"]")
type ShareInfos struct {
Shares []ShareInfo `json:"shares"`
}
resp, err := dsm.sendRequest("", &ShareInfos{}, params, "webapi/entry.cgi")
if err != nil {
return nil, err
}
infos, ok := resp.Data.(*ShareInfos)
if !ok {
return nil, fmt.Errorf("Failed to assert response to %T", &ShareInfos{})
}
return infos.Shares, nil
}
func (dsm *DSM) ShareCreate(spec ShareCreateSpec) error {
params := url.Values{}
params.Add("api", "SYNO.Core.Share")
params.Add("method", "create")
params.Add("version", "1")
params.Add("name", strconv.Quote(spec.Name))
js, err := json.Marshal(spec.ShareInfo)
if err != nil {
return err
}
params.Add("shareinfo", string(js))
// response : {"data":{"name":"Share-2"},"success":true}
_, err = dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
if err != nil {
return err
}
return nil
}
func (dsm *DSM) ShareDelete(shareName string) error {
params := url.Values{}
params.Add("api", "SYNO.Core.Share")
params.Add("method", "delete")
params.Add("version", "1")
params.Add("name", strconv.Quote(shareName))
// response : {"success":true}
_, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
if err != nil {
return err
}
return nil
}

67
pkg/dsm/webapi/storage.go Normal file
View File

@@ -0,0 +1,67 @@
// Copyright 2021 Synology Inc.
package webapi
import (
"fmt"
"strconv"
"net/url"
)
type VolInfo struct {
Name string `json:"display_name"`
Path string `json:"volume_path"`
Status string `json:"status"`
FsType string `json:"fs_type"`
Size string `json:"size_total_byte"`
Free string `json:"size_free_byte"`
Container string `json:"container"`
Location string `json:"location"`
}
func (dsm *DSM) VolumeList() ([]VolInfo, error) {
params := url.Values{}
params.Add("api", "SYNO.Core.Storage.Volume")
params.Add("method", "list")
params.Add("version", "1")
params.Add("offset", "0")
params.Add("limit", "-1")
params.Add("location", "all")
type VolInfos struct {
Vols []VolInfo `json:"volumes"`
}
resp, err := dsm.sendRequest("", &VolInfos{}, params, "webapi/entry.cgi")
if err != nil {
return nil, err
}
volInfos, ok := resp.Data.(*VolInfos)
if !ok {
return nil, fmt.Errorf("Failed to assert response to %T", &VolInfos{})
}
return volInfos.Vols, nil
}
func (dsm *DSM) VolumeGet(name string) (VolInfo, error) {
params := url.Values{}
params.Add("api", "SYNO.Core.Storage.Volume")
params.Add("method", "get")
params.Add("version", "1")
params.Add("volume_path", strconv.Quote(name))
type Info struct {
Volume VolInfo `json:"volume"`
}
info := Info{}
_, err := dsm.sendRequest("", &info, params, "webapi/entry.cgi")
if err != nil {
return VolInfo{}, err
}
return info.Volume, nil
}

View File

@@ -0,0 +1,57 @@
// Copyright 2021 Synology Inc.
package webapi
import (
"fmt"
"net/url"
)
type DsmInfo struct {
Hostname string `json:"hostname"`
}
type DsmSysInfo struct {
Model string `json:"model"`
FirmwareVer string `json:"firmware_ver"`
Serial string `json:"serial"`
}
func (dsm *DSM) DsmInfoGet() (*DsmInfo, error) {
params := url.Values{}
params.Add("api", "SYNO.Core.System")
params.Add("method", "info")
params.Add("version", "1")
params.Add("type", "network")
resp, err := dsm.sendRequest("", &DsmInfo{}, params, "webapi/entry.cgi")
if err != nil {
return nil, err
}
dsmInfo, ok := resp.Data.(*DsmInfo)
if !ok {
return nil, fmt.Errorf("Failed to assert response to %T", &DsmInfo{})
}
return dsmInfo, nil
}
func (dsm *DSM) DsmSystemInfoGet() (*DsmSysInfo, error) {
params := url.Values{}
params.Add("api", "SYNO.Core.System")
params.Add("method", "info")
params.Add("version", "1")
resp, err := dsm.sendRequest("", &DsmSysInfo{}, params, "webapi/entry.cgi")
if err != nil {
return nil, err
}
dsmInfo, ok := resp.Data.(*DsmSysInfo)
if !ok {
return nil, fmt.Errorf("Failed to assert response to %T", &DsmSysInfo{})
}
return dsmInfo, nil
}

1
pkg/dsm/webapi/utils.go Normal file
View File

@@ -0,0 +1 @@
package webapi

View File

@@ -0,0 +1,29 @@
// Copyright 2021 Synology Inc.
package interfaces
import (
"github.com/SynologyOpenSource/synology-csi/pkg/dsm/common"
"github.com/SynologyOpenSource/synology-csi/pkg/dsm/webapi"
"github.com/SynologyOpenSource/synology-csi/pkg/models"
)
// An interface for DSM service
type IDsmService interface {
AddDsm(client common.ClientInfo) error
RemoveAllDsms()
GetDsm(ip string) (*webapi.DSM, error)
GetDsmsCount() int
ListDsmVolumes(ip string) ([]webapi.VolInfo, error)
CreateVolume(spec *models.CreateK8sVolumeSpec) (webapi.LunInfo, string, error)
DeleteVolume(volumeId string) error
ListVolumes() []*models.ListK8sVolumeRespSpec
GetVolume(lunUuid string) *models.ListK8sVolumeRespSpec
ExpandLun(lunUuid string, newSize int64) error
CreateSnapshot(spec *models.CreateK8sVolumeSnapshotSpec) (string, error)
DeleteSnapshot(snapshotUuid string) error
ListAllSnapshots() ([]webapi.SnapshotInfo, error)
ListSnapshots(lunUuid string) ([]webapi.SnapshotInfo, error)
GetSnapshot(snapshotUuid string) (webapi.SnapshotInfo, error)
}

98
pkg/logger/logger.go Normal file
View File

@@ -0,0 +1,98 @@
// Copyright 2021 Synology Inc.
package logger
import (
"os"
"runtime"
"strings"
"strconv"
"time"
"github.com/sirupsen/logrus"
nested "github.com/antonfisher/nested-logrus-formatter"
)
var WebapiDebug = false
const (
DefaultLogLevel = logrus.InfoLevel
DefaultTimestampFormat = time.RFC3339
)
type CallerHook struct {
FileNameAndLine string
Skip int
}
func (hook *CallerHook) Fire(entry *logrus.Entry) error {
fileName, line := findCaller(hook.Skip)
entry.Data[hook.FileNameAndLine] = fileName + ":" + strconv.Itoa(line) // <filename>:<line>
return nil
}
func (hook *CallerHook) Levels() []logrus.Level {
return logrus.AllLevels
}
func NewCallerHook() logrus.Hook {
hook := CallerHook{
FileNameAndLine: "filePath",
Skip: 5,
}
return &hook
}
func getCaller(skip int) (string, int) {
_, file, line, ok := runtime.Caller(skip)
if !ok{
return "", 0
}
n := 0
for i := len(file) - 1; i > 0; i-- {
if string(file[i]) == "/" {
n++
if n >= 2 {
file = file[i+1:]
break
}
}
}
return file, line
}
func findCaller(skip int) (string, int) {
var file string
var line int
for i := 0; i < 10; i++ {
file, line = getCaller(skip+i)
// if called by logrus functions, continue finding the upper caller
if !strings.HasPrefix(file, "logrus"){
break
}
}
return file, line
}
func setLogLevel(logLevel string) { // debug, info, warn, error, fatal
level, err := logrus.ParseLevel(logLevel)
if err == nil {
logrus.SetLevel(level)
} else {
logrus.SetLevel(DefaultLogLevel)
}
return
}
func Init(logLevel string) {
logrus.AddHook(NewCallerHook())
logrus.SetOutput(os.Stdout)
setLogLevel(logLevel)
logrus.SetFormatter(&nested.Formatter{
HideKeys: true,
TimestampFormat: DefaultTimestampFormat,
ShowFullLevel: true,
NoColors: true,
})
}

22
pkg/models/dsm.go Normal file
View File

@@ -0,0 +1,22 @@
// Copyright 2021 Synology Inc.
package models
const (
K8sCsiName = "Kubernetes CSI"
// ISCSI definitions
FsTypeExt4 = "ext4"
FsTypeBtrfs = "btrfs"
LunTypeFile = "FILE"
LunTypeThin = "THIN"
LunTypeAdv = "ADV"
LunTypeBlun = "BLUN" // thin provision, mapped to type 263
LunTypeBlunThick = "BLUN_THICK" // thick provision, mapped to type 259
MaxIqnLen = 128
// CSI definitions
TargetPrefix = "k8s-csi"
LunPrefix = "k8s-csi"
IqnPrefix = "iqn.2000-01.com.synology:"
)

View File

@@ -0,0 +1,36 @@
// Copyright 2021 Synology Inc.
package models
import (
"github.com/SynologyOpenSource/synology-csi/pkg/dsm/webapi"
)
type CreateK8sVolumeSpec struct {
DsmIp string
K8sVolumeName string
LunName string
Location string
Size int64
Type string
ThinProvisioning bool
TargetName string
TargetIqn string
MultipleSession bool
SourceSnapshotId string
SourceVolumeId string
}
type ListK8sVolumeRespSpec struct {
DsmIp string
Lun webapi.LunInfo
Target webapi.TargetInfo
}
type CreateK8sVolumeSnapshotSpec struct {
K8sVolumeId string
SnapshotName string
Description string
TakenBy string
IsLocked bool
}

43
pkg/utils/error.go Normal file
View File

@@ -0,0 +1,43 @@
// Copyright 2021 Synology Inc.
package utils
type OutOfFreeSpaceError string
type AlreadyExistError string
type LunReachMaxCountError string
type TargetReachMaxCountError string
type NoSuchSnapshotError string
type BadLunTypeError string
type SnapshotReachMaxCountError string
type IscsiDefaultError string
func (_ OutOfFreeSpaceError) Error() string {
return "Out of free space"
}
func (_ AlreadyExistError) Error() string {
return "Already Existed"
}
func (_ LunReachMaxCountError) Error() string {
return "Number of LUN reach limit"
}
func (_ TargetReachMaxCountError) Error() string {
return "Number of target reach limit"
}
func (_ NoSuchSnapshotError) Error() string {
return "No such snapshot uuid"
}
func (_ BadLunTypeError) Error() string {
return "Bad LUN type"
}
func (_ SnapshotReachMaxCountError) Error() string {
return "Number of snapshot reach limit"
}
func (_ IscsiDefaultError) Error() string {
return "Default ISCSI error"
}

33
pkg/utils/utils.go Normal file
View File

@@ -0,0 +1,33 @@
// Copyright 2021 Synology Inc.
package utils
import (
"fmt"
"net"
"strings"
)
const UNIT_GB = 1024 * 1024 * 1024
func StringToBoolean(value string) bool {
value = strings.ToLower(value)
return value == "yes" || value == "true" || value == "1"
}
// Haven't supported IPv6 yet.
func LookupIPv4(name string) ([]string, error) {
ips, _ := net.LookupIP(name)
retIps := []string{}
for _, ip := range ips {
if ipv4 := ip.To4(); ipv4 != nil {
retIps = append(retIps, ipv4.String())
}
}
if len(retIps) > 0 {
return retIps, nil
}
return nil, fmt.Errorf("Failed to LookupIPv4 by local resolver for: %s", name)
}

81
scripts/deploy.sh Executable file
View File

@@ -0,0 +1,81 @@
#!/bin/bash
plugin_name="csi.san.synology.com"
deploy_k8s_version="v1.19"
SCRIPT_PATH="$(realpath "$0")"
SOURCE_PATH="$(realpath "$(dirname "${SCRIPT_PATH}")"/../)"
config_file="${SOURCE_PATH}/config/client-info.yml"
plugin_dir="/var/lib/kubelet/plugins/$plugin_name"
# 1. Build
csi_build(){
echo "==== Build synology-csi .... ===="
source "$SOURCE_PATH"/build.sh
}
# 2. Install
csi_install(){
echo "==== Creates namespace and secrets, then installs synology-csi ===="
kubectl create ns synology-csi
kubectl create secret -n synology-csi generic client-info-secret --from-file="$config_file"
if [ ! -d "$plugin_dir" ]; then
mkdir -p $plugin_dir
fi
kubectl apply -f "$SOURCE_PATH"/deploy/kubernetes/$deploy_k8s_version
if [ "$basic_mode" == false ]; then
kubectl apply -f "$SOURCE_PATH"/deploy/kubernetes/$deploy_k8s_version/snapshotter
fi
}
print_usage(){
echo "Usage:"
echo " deploy.sh run build and install"
echo " deploy.sh [command] specify an action to be performed"
echo "Available Commands:"
echo " build build docker image only"
echo " install [flag] install csi plugin with the specified flag"
echo " help show help"
echo "Available Flags:"
echo " -a, --all deploy csi plugin and snapshotter"
echo " -b, --basic deploy basic csi plugin only"
echo "Examples:"
echo " deploy.sh run"
echo " deploy.sh install --basic"
}
basic_mode=false
parse_flags(){
case "$1" in
-a|--all)
;;
-b|--basic)
basic_mode=true
;;
*)
print_usage
exit 1
;;
esac
}
case "$1" in
build)
csi_build
;;
install)
parse_flags "$2"
csi_install
;;
run)
csi_build
csi_install
;;
*)
print_usage
exit 1
;;
esac

12
scripts/uninstall.sh Executable file
View File

@@ -0,0 +1,12 @@
#!/bin/bash
echo "Uninstalling synology-csi pods ..."
plugin_name="csi.san.synology.com"
deploy_k8s_version="v1.19"
SCRIPT_PATH="$(realpath "$0")"
SOURCE_PATH="$(realpath "$(dirname "$SCRIPT_PATH")"/../)"
kubectl delete -f "$SOURCE_PATH"/deploy/kubernetes/$deploy_k8s_version/snapshotter --ignore-not-found
kubectl delete -f "$SOURCE_PATH"/deploy/kubernetes/$deploy_k8s_version --ignore-not-found
echo "End of synology-csi uninstallation."

59
synocli/cmd/dsm.go Normal file
View File

@@ -0,0 +1,59 @@
/*
* Copyright 2021 Synology Inc.
*/
package cmd
import (
"fmt"
"os"
"github.com/spf13/cobra"
"github.com/SynologyOpenSource/synology-csi/pkg/dsm/webapi"
)
var https = false
var port = -1
var cmdDsm = &cobra.Command{
Use: "dsm",
Short: "dsm",
Long: `dsm`,
Run: func(cmd *cobra.Command, args []string) {
cmd.Help()
},
}
var cmdDsmLogin = &cobra.Command{
Use: "login <ip> <username> <password>",
Short: "login dsm",
Args: cobra.MinimumNArgs(3),
Run: func(cmd *cobra.Command, args []string) {
var defaultPort = 5000
if https {
defaultPort = 5001
}
if port != -1 {
defaultPort = port
}
dsmApi := &webapi.DSM{
Ip: args[0],
Username: args[1],
Password: args[2],
Port: defaultPort,
Https: https,
}
err := dsmApi.Login()
if err != nil {
fmt.Println(err)
os.Exit(1)
}
},
}
func init() {
cmdDsm.AddCommand(cmdDsmLogin)
cmdDsmLogin.PersistentFlags().BoolVar(&https, "https", false, "Use HTTPS to login DSM")
cmdDsmLogin.PersistentFlags().IntVarP(&port, "port", "p", -1, "Use assigned port to login DSM")
}

23
synocli/cmd/root.go Normal file
View File

@@ -0,0 +1,23 @@
/*
* Copyright 2021 Synology Inc.
*/
package cmd
import (
"github.com/spf13/cobra"
)
var rootCmd = &cobra.Command{
Use: "synocli",
Run: func(cmd *cobra.Command, args []string) {
cmd.Help()
},
}
func init() {
rootCmd.AddCommand(cmdDsm)
}
func Execute() {
rootCmd.Execute()
}

13
synocli/main.go Normal file
View File

@@ -0,0 +1,13 @@
/*
* Copyright 2021 Synology Inc.
*/
package main
import (
"github.com/SynologyOpenSource/synology-csi/synocli/cmd"
)
func main() {
cmd.Execute()
}

View File

@@ -0,0 +1,74 @@
package sanitytest
import (
"fmt"
"io/ioutil"
"os"
"testing"
sanity "github.com/kubernetes-csi/csi-test/v4/pkg/sanity"
"github.com/SynologyOpenSource/synology-csi/pkg/driver"
"github.com/SynologyOpenSource/synology-csi/pkg/dsm/common"
"github.com/SynologyOpenSource/synology-csi/pkg/dsm/service"
)
const (
ConfigPath = "./../../config/client-info.yml"
)
func TestSanity(t *testing.T) {
nodeID := "CSINode"
endpointFile, err := ioutil.TempFile("", "csi-gcs.*.sock")
if err != nil {
t.Fatal(err)
}
defer os.Remove(endpointFile.Name())
stagingPath, err := ioutil.TempDir("", "csi-gcs-staging")
if err != nil {
t.Fatal(err)
}
os.Remove(stagingPath)
targetPath, err := ioutil.TempDir("", "csi-gcs-target")
if err != nil {
t.Fatal(err)
}
os.Remove(targetPath)
info, err := common.LoadConfig(ConfigPath)
if err != nil {
t.Fatal(fmt.Sprintf("Failed to read config: %v", err))
}
dsmService := service.NewDsmService()
for _, client := range info.Clients {
err := dsmService.AddDsm(client)
if err != nil {
fmt.Printf("Failed to add DSM: %s, error: %v\n", client.Host, err)
}
}
if dsmService.GetDsmsCount() == 0 {
t.Fatal("No available DSM.")
}
defer dsmService.RemoveAllDsms()
endpoint := "unix://" + endpointFile.Name()
drv, err := driver.NewControllerAndNodeDriver(nodeID, endpoint, dsmService)
if err != nil {
t.Fatal(fmt.Sprintf("Failed to create driver: %v\n", err))
}
drv.Activate()
// Set configuration options as needed
testConfig := sanity.NewTestConfig()
testConfig.TargetPath = targetPath
testConfig.StagingPath = stagingPath
testConfig.Address = endpoint
// Run test
sanity.Test(t, testConfig)
}