Merge pull request apalia#1 from swisstxt/feature/cargo-55

Feature/cargo 55
ubeeo · Oct 9, 2021 · 55bfbbf · 55bfbbf
2 parents 81a9b2b + 111215e
commit 55bfbbf
Show file tree

Hide file tree

Showing 18 changed files with 543 additions and 128 deletions.
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -34,16 +34,16 @@ jobs:
       - name: Log into registry
         uses: docker/login-action@v1
         with:
-          registry: quay.io
-          username: apalia+github_cloudstack_csi_driver
-          password: ${{ secrets.QUAY_TOKEN }}
+          registry: ${{ secrets.REGISTRY_CARGO_SYSTEM_URL }}
+          username: ${{ secrets.REGISTRY_CARGO_SYSTEM_USERNAME }}
+          password: ${{ secrets.REGISTRY_CARGO_SYSTEM_PASSWORD }}
 
       - name: Push master
         if: github.ref == 'refs/heads/master'
         run: |
           for img in $IMAGES; do
-              docker tag ${img} ${REGISTRY_NAME}/${img}:master
-              docker push ${REGISTRY_NAME}/${img}:master
+              docker tag ${img} ${{ secrets.REGISTRY_CARGO_URL}}/${img}:master
+              docker push ${{ secrets.REGISTRY_CARGO_URL}}/${img}:master
           done
 
       - name: Push tagged release
@@ -53,8 +53,8 @@ jobs:
           VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,' | sed -e 's/^v//')
 
           for img in $IMAGES; do
-              docker tag ${img} ${REGISTRY_NAME}/${img}:${VERSION}
-              docker push ${REGISTRY_NAME}/${img}:${VERSION}
+              docker tag ${img} ${{ secrets.REGISTRY_CARGO_URL}}/${img}:${VERSION}
+              docker push ${{ secrets.REGISTRY_CARGO_URL}}/${img}:${VERSION}
           done
 
       - name: Upload cloudstack-csi-sc-syncer artifact
@@ -87,9 +87,9 @@ jobs:
           echo "---" >> manifest.yaml
           cat deploy/k8s/csidriver.yaml >> manifest.yaml
           echo "---" >> manifest.yaml
-          sed -E "s|image: +cloudstack-csi-driver|image: ${REGISTRY_NAME}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/controller-deployment.yaml >> manifest.yaml
+          sed -E "s|image: +cloudstack-csi-driver|image: ${{ secrets.REGISTRY_CARGO_URL}}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/controller-deployment.yaml >> manifest.yaml
           echo "---" >> manifest.yaml
-          sed -E "s|image: +cloudstack-csi-driver|image: ${REGISTRY_NAME}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/node-daemonset.yaml >> manifest.yaml
+          sed -E "s|image: +cloudstack-csi-driver|image: ${{ secrets.REGISTRY_CARGO_URL}}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/node-daemonset.yaml >> manifest.yaml
 
       - name: Create Release
         id: create_release

diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,6 @@
 /bin
 /test/e2e/e2e.test
 /test/e2e/ginkgo
+/cloudstack.ini
+/.idea
 cloud-config
diff --git a/README.md b/README.md
@@ -40,6 +40,7 @@ api-url = <CloudStack API URL>
 api-key = <CloudStack API Key>
 secret-key = <CloudStack API Secret>
 ssl-no-verify = <Disable SSL certificate validation: true or false (optional)>
+project-id = <project ID>
 ```
 
 Create a secret named `cloudstack-secret` in namespace `kube-system`:
@@ -51,6 +52,18 @@ kubectl create secret generic \
   cloudstack-secret
 ```
 
+Set the correct hypervisor in the DaemonSet Env Vars:
+```
+            - name: NODE_HYPERVISOR
+              value: vmware
+```
+
+You can manually set the maximal attachable number of block volumes per node:
+```
+            - name: NODE_MAX_BLOCK_VOLUMES
+              value: "15" #Default value is 10 volumes per node
+```
+
 If you have also deployed the [CloudStack Kubernetes Provider](https://github.com/apache/cloudstack-kubernetes-provider),
 you may use the same secret for both tools.
 
@@ -88,11 +101,19 @@ disk offerings to Kubernetes storage classes.
 
 Example:
 
-```
+```bash
 kubectl apply -f ./examples/k8s/pvc.yaml
 kubectl apply -f ./examples/k8s/pod.yaml
 ```
 
+#### Reusing volumes
+
+1. Patch PV `reclaimPolicy` with `kubectl patch pv my-pv-name -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'`
+2. Delete Old Pod and PVC
+3. Patch PV `claimRef` with `kubectl patch pv my-pv-name -p '{"spec":{"claimRef": null}}'`
+4. Create new Pod and PVC with existing claimName `.spec.claimRef.name = my-pv-name`
+
+
 ## Building
 
 To build the driver binary:

diff --git a/clean-scsi-bus.sh b/clean-scsi-bus.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+echo 1 > /sys/class/scsi_device/0\:0\:$1\:0/device/delete
diff --git a/cmd/cloudstack-csi-driver/Dockerfile b/cmd/cloudstack-csi-driver/Dockerfile
@@ -2,7 +2,7 @@ FROM alpine:3.14.0
 
 LABEL \
     org.opencontainers.image.description="CloudStack CSI driver" \
-    org.opencontainers.image.source="https://github.com/apalia/cloudstack-csi-driver/"
+    org.opencontainers.image.source="https://github.com/swisstxt/cloudstack-csi-driver/"
 
 RUN apk add --no-cache \
     ca-certificates \
@@ -11,7 +11,13 @@ RUN apk add --no-cache \
     # Provides mkfs.xfs
     xfsprogs \
     # Provides blkid, also used by k8s.io/mount-utils
-    blkid
+    blkid \
+    eudev \
+    bash
 
 COPY ./bin/cloudstack-csi-driver /cloudstack-csi-driver
+COPY rescan-scsi-bus.sh /usr/bin/
+RUN chmod +x /usr/bin/rescan-scsi-bus.sh
+COPY clean-scsi-bus.sh /usr/bin/
+RUN chmod +x /usr/bin/clean-scsi-bus.sh
 ENTRYPOINT ["/cloudstack-csi-driver"]
diff --git a/deploy/k8s/node-daemonset.yaml b/deploy/k8s/node-daemonset.yaml
@@ -2,101 +2,134 @@ apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: cloudstack-csi-node
-  namespace: kube-system
 spec:
+  revisionHistoryLimit: 10
   selector:
     matchLabels:
       app.kubernetes.io/name: cloudstack-csi-node
-  updateStrategy:
-    type: RollingUpdate
   template:
     metadata:
       labels:
         app.kubernetes.io/name: cloudstack-csi-node
         app.kubernetes.io/part-of: cloudstack-csi-driver
     spec:
-      nodeSelector:
-        kubernetes.io/os: linux
-      tolerations:
-        - effect: NoExecute
-          operator: Exists
-        - effect: NoSchedule
-          operator: Exists
-
       containers:
-        - name: cloudstack-csi-node
-          image: cloudstack-csi-driver
-          imagePullPolicy: Always
-          args:
-            - "-endpoint=$(CSI_ENDPOINT)"
-            - "-cloudstackconfig=/etc/cloudstack-csi-driver/cloud-config"
-            - "-nodeName=$(NODE_NAME)"
-            - "-debug"
+        - args:
+            - -endpoint=$(CSI_ENDPOINT)
+            - -cloudstackconfig=/etc/cloudstack-csi-driver/cloudstack.ini
+            - -nodeName=$(NODE_NAME)
+            - -debug
           env:
             - name: CSI_ENDPOINT
               value: unix:///csi/csi.sock
+            - name: NODE_HYPERVISOR
+              value: vmware
+            - name: NODE_MAX_BLOCK_VOLUMES
+              value: "10"
             - name: NODE_NAME
               valueFrom:
                 fieldRef:
+                  apiVersion: v1
                   fieldPath: spec.nodeName
+          image: registry.swisstxt.ch/stxt-proj-cargo-system/cloudstack-csi-driver:latest
+          imagePullPolicy: Always
+          name: cloudstack-csi-node
+          resources: {}
           securityContext:
             privileged: true
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
           volumeMounts:
-            - name: plugin-dir
-              mountPath: /csi
-            - name: kubelet-dir
-              mountPath: /var/lib/kubelet
-              # needed so that any mounts setup inside this container are
-              # propagated back to the host machine.
-              mountPropagation: Bidirectional
-            - name: device-dir
-              mountPath: /dev
-            - name: cloud-init-dir
-              mountPath: /run/cloud-init/
-            - name: cloudstack-conf
-              mountPath: /etc/cloudstack-csi-driver
-
-        - name: node-driver-registrar
-          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1
-          imagePullPolicy: IfNotPresent
-          args:
-            - "--csi-address=$(ADDRESS)"
-            - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
-            - "--v=5"
+          - mountPath: /csi
+            name: plugin-dir
+          - mountPath: /var/lib/kubelet
+            mountPropagation: Bidirectional
+            name: kubelet-dir
+          - mountPath: /dev
+            name: device-dir
+          - mountPath: /run/cloud-init/
+            name: cloud-init-dir
+          - mountPath: /etc/cloudstack-csi-driver
+            name: cloudstack-conf
+          - mountPath: /sys/class/scsi_host/
+            name: sys-class-scsi-host-dir
+          - mountPath: /sys/class/scsi_device/
+            name: sys-class-scsi-device-dir
+          - mountPath: /sys/devices
+            name: sys-devices
+        - args:
+            - --csi-address=$(ADDRESS)
+            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+            - --v=5
           env:
             - name: ADDRESS
               value: /csi/csi.sock
             - name: DRIVER_REG_SOCK_PATH
               value: /var/lib/kubelet/plugins/csi.cloudstack.apache.org/csi.sock
+          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1
+          imagePullPolicy: IfNotPresent
+          name: node-driver-registrar
+          resources: {}
           securityContext:
             privileged: true
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
           volumeMounts:
-            - name: plugin-dir
-              mountPath: /csi
-            - name: registration-dir
-              mountPath: /registration
-
+            - mountPath: /csi
+              name: plugin-dir
+            - mountPath: /registration
+              name: registration-dir
+      dnsPolicy: ClusterFirst
+      nodeSelector:
+        kubernetes.io/os: linux
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      serviceAccount: cloudstack-csi-node
+      serviceAccountName: cloudstack-csi-node
+      terminationGracePeriodSeconds: 30
+      tolerations:
+        - effect: NoExecute
+          operator: Exists
+        - effect: NoSchedule
+          operator: Exists
       volumes:
-        - name: plugin-dir
-          hostPath:
+        - hostPath:
             path: /var/lib/kubelet/plugins/csi.cloudstack.apache.org/
             type: DirectoryOrCreate
-        - name: kubelet-dir
-          hostPath:
+          name: plugin-dir
+        - hostPath:
             path: /var/lib/kubelet
             type: Directory
-        - name: device-dir
-          hostPath:
+          name: kubelet-dir
+        - hostPath:
             path: /dev
             type: Directory
-        - name: registration-dir
-          hostPath:
+          name: device-dir
+        - hostPath:
             path: /var/lib/kubelet/plugins_registry
             type: Directory
-        - name: cloud-init-dir
-          hostPath:
+          name: registration-dir
+        - hostPath:
             path: /run/cloud-init/
             type: Directory
+          name: cloud-init-dir
+        - hostPath:
+            path: /sys/class/scsi_host
+            type: Directory
+          name: sys-class-scsi-host-dir
+        - hostPath:
+            path: /sys/class/scsi_device
+            type: Directory
+          name: sys-class-scsi-device-dir
+        - hostPath:
+            path: /sys/devices
+            type: Directory
+          name: sys-devices
         - name: cloudstack-conf
           secret:
-            secretName: cloudstack-secret
+            defaultMode: 420
+            secretName: csi-cloudstack-secret
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
diff --git a/go.mod b/go.mod
@@ -9,6 +9,7 @@ require (
 	github.com/hashicorp/go-uuid v1.0.2
 	github.com/kubernetes-csi/csi-test/v4 v4.2.0
 	go.uber.org/zap v1.16.0
+	golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d
 	golang.org/x/text v0.3.6
 	google.golang.org/genproto v0.0.0-20210726200206-e7812ac95cc0 // indirect
 	google.golang.org/grpc v1.39.0

diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go
@@ -16,9 +16,13 @@ type Interface interface {
 
 	ListZonesID(ctx context.Context) ([]string, error)
 
+	GetDomainID(ctx context.Context) (string, error)
+	GetProjectID() string
+
 	GetVolumeByID(ctx context.Context, volumeID string) (*Volume, error)
 	GetVolumeByName(ctx context.Context, name string) (*Volume, error)
-	CreateVolume(ctx context.Context, diskOfferingID, zoneID, name string, sizeInGB int64) (string, error)
+	ListVolumesForVM(ctx context.Context, virtualMachineID, projectID string) ([]*Volume, error)
+	CreateVolume(ctx context.Context, diskOfferingID, projectID, domainID, zoneID, name string, sizeInGB int64) (string, error)
 	DeleteVolume(ctx context.Context, id string) error
 	AttachVolume(ctx context.Context, volumeID, vmID string) (string, error)
 	DetachVolume(ctx context.Context, volumeID string) error
@@ -37,6 +41,7 @@ type Volume struct {
 
 	VirtualMachineID string
 	DeviceID         string
+	Hypervisor       string
 }
 
 // VM represents a CloudStack Virtual Machine.
@@ -54,10 +59,11 @@ var (
 // client is the implementation of Interface.
 type client struct {
 	*cloudstack.CloudStackClient
+	ProjectID string
 }
 
 // New creates a new cloud connector, given its configuration.
 func New(config *Config) Interface {
 	csClient := cloudstack.NewAsyncClient(config.APIURL, config.APIKey, config.SecretKey, config.VerifySSL)
-	return &client{csClient}
+	return &client{csClient, config.ProjectID}
 }
diff --git a/pkg/cloud/config.go b/pkg/cloud/config.go
@@ -12,6 +12,7 @@ type Config struct {
 	APIKey    string
 	SecretKey string
 	VerifySSL bool
+	ProjectID string
 }
 
 // csConfig wraps the config for the CloudStack cloud provider.
@@ -42,5 +43,6 @@ func ReadConfig(configFilePath string) (*Config, error) {
 		APIKey:    cfg.Global.APIKey,
 		SecretKey: cfg.Global.SecretKey,
 		VerifySSL: cfg.Global.SSLNoVerify,
+		ProjectID: cfg.Global.ProjectID,
 	}, nil
 }
diff --git a/pkg/cloud/fake/fake.go b/pkg/cloud/fake/fake.go
@@ -73,7 +73,7 @@ func (f *fakeConnector) GetVolumeByName(ctx context.Context, name string) (*clou
 	return nil, cloud.ErrNotFound
 }
 
-func (f *fakeConnector) CreateVolume(ctx context.Context, diskOfferingID, zoneID, name string, sizeInGB int64) (string, error) {
+func (f *fakeConnector) CreateVolume(ctx context.Context, diskOfferingID, projectID, domainID, zoneID, name string, sizeInGB int64) (string, error) {
 	id, _ := uuid.GenerateUUID()
 	vol := cloud.Volume{
 		ID:             id,
@@ -101,3 +101,11 @@ func (f *fakeConnector) AttachVolume(ctx context.Context, volumeID, vmID string)
 }
 
 func (f *fakeConnector) DetachVolume(ctx context.Context, volumeID string) error { return nil }
+
+func (f *fakeConnector) GetDomainID(ctx context.Context) (string, error) {
+	return "domain", nil
+}
+
+func (f *fakeConnector) GetProjectID() string {
+	return "test"
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		#!/bin/sh
		echo 1 > /sys/class/scsi_device/0\:0\:$1\:0/device/delete