diff --git a/cluster/images/obs-csi-plugin/Dockerfile b/cluster/images/obs-csi-plugin/Dockerfile index 593ac405..b278f2d8 100644 --- a/cluster/images/obs-csi-plugin/Dockerfile +++ b/cluster/images/obs-csi-plugin/Dockerfile @@ -5,16 +5,13 @@ LABEL description="HuaweiCloud CSI Plugin" WORKDIR /obs-csi -COPY obsfs_CentOS7.6_amd64.tar.gz ./ -COPY obsfs_Ubuntu16.04_amd64.tar.gz ./ COPY entrypoint.sh entrypoint.sh COPY nsenter /nsenter COPY csi-connector-server ./ COPY csi-connector.service ./ COPY obs-csi-plugin obs-csi-plugin -COPY install_obsfs.sh ./ +COPY install_s3fs.sh ./ COPY stop-server.sh ./ -COPY huaweicloud-obs-obsfs.tar.gz ./ RUN chmod +x obs-csi-plugin RUN chmod +x entrypoint.sh diff --git a/cluster/images/obs-csi-plugin/csi-connector-server.go b/cluster/images/obs-csi-plugin/csi-connector-server.go index 62daffbb..26f1108f 100644 --- a/cluster/images/obs-csi-plugin/csi-connector-server.go +++ b/cluster/images/obs-csi-plugin/csi-connector-server.go @@ -25,7 +25,7 @@ const ( region = "region" cloud = "cloud" credential = "credential" - defaultOpts = "-o big_writes -o max_write=131072 -o use_ino" + defaultOpts = "-o nonempty -o big_writes -o max_write=131072" ) type ResponseBody struct { @@ -119,7 +119,7 @@ func main() { //nolint:errcheck flag.CommandLine.Parse([]string{}) - initObsfsUtil() + initS3fsUtil() if checkFileExists(obs.SocketPath) { if err := os.Remove(obs.SocketPath); err != nil { log.Fatalf("Failed to remove path: %s, err: %v", obs.SocketPath, err) @@ -153,26 +153,27 @@ func mountHandler(parameters map[string]string) error { if parameters[cloud] == "prod-cloud-ocb.orange-business.com" { obsName = "oss" } - mountOpts := parameters[mountFlags] if mountOpts == "" { mountOpts = defaultOpts } + options := []string{ - "obsfs", parameters[bucketName], parameters[targetPath], - fmt.Sprintf("-o url=%s.%s.%s", obsName, parameters[region], parameters[cloud]), - fmt.Sprintf("-o passwd_file=%s", credentialFile), - mountOpts, + "-o", + fmt.Sprintf("url=https://%s.%s.%s", obsName, parameters[region], parameters[cloud]), + "-o", + fmt.Sprintf("passwd_file=%s", credentialFile), } + mntOptsArr := strings.Split(mountOpts, " ") + options = append(options, mntOptsArr...) - cmd := exec.Command("sh", "-c") - cmd.Args = append(cmd.Args, strings.Join(options, " ")) + cmd := exec.Command("s3fs", options...) out, err := cmd.CombinedOutput() if err != nil { - log.Errorf("failed to mount CMD: obsfs %s, output: %s, error: %v", strings.Join(options, " "), string(out), err) - return fmt.Errorf("failed to mount CMD: obsfs %s, output: %s, error: %v", strings.Join(options, " "), string(out), err) + log.Errorf("failed to mount bucket to node, CMD: s3fs %s, output: %s, error: %v", strings.Join(options, " "), string(out), err) + return fmt.Errorf("failed to mount bucket to node, CMD: s3fs %s, output: %s, error: %v", strings.Join(options, " "), string(out), err) } log.Infof("success to mount CMD: %s", strings.Join(options, " ")) return nil @@ -186,11 +187,11 @@ func deleteCredential(credential string) { } } -func initObsfsUtil() { - cmd := fmt.Sprintf("sh %s/install_obsfs.sh >> %s/connector.log 2>&1 &", credentialDir, credentialDir) +func initS3fsUtil() { + cmd := fmt.Sprintf("sh %s/install_s3fs.sh > %s/connector.log 2>&1 &", credentialDir, credentialDir) out, err := exec.Command("sh", "-c", cmd).CombinedOutput() - log.Infof("install obsfs %s", string(out)) + log.Infof("install s3fs %s", string(out)) if err != nil { - log.Errorf("error install obsfs: %s", err) + log.Errorf("error install s3fs: %s", err) } } diff --git a/cluster/images/obs-csi-plugin/entrypoint.sh b/cluster/images/obs-csi-plugin/entrypoint.sh index d1bb5f1c..085f4b61 100644 --- a/cluster/images/obs-csi-plugin/entrypoint.sh +++ b/cluster/images/obs-csi-plugin/entrypoint.sh @@ -3,10 +3,7 @@ HOST_CMD="/nsenter --mount=/proc/1/ns/mnt" mkdir -p /var/lib/csi/ -cp -f /obs-csi/huaweicloud-obs-obsfs.tar.gz /var/lib/csi/huaweicloud-obs-obsfs.tar.gz -cp -f /obs-csi/obsfs_CentOS7.6_amd64.tar.gz /var/lib/csi/obsfs_CentOS7.6_amd64.tar.gz -cp -f /obs-csi/obsfs_Ubuntu16.04_amd64.tar.gz /var/lib/csi/obsfs_Ubuntu16.04_amd64.tar.gz -cp -f /obs-csi/install_obsfs.sh /var/lib/csi/install_obsfs.sh +cp -f /obs-csi/install_s3fs.sh /var/lib/csi/install_s3fs.sh echo "Starting install obs csi-connector-server...." $HOST_CMD systemctl stop csi-connector.service diff --git a/cluster/images/obs-csi-plugin/huaweicloud-obs-obsfs.tar.gz b/cluster/images/obs-csi-plugin/huaweicloud-obs-obsfs.tar.gz deleted file mode 100644 index c6f508be..00000000 Binary files a/cluster/images/obs-csi-plugin/huaweicloud-obs-obsfs.tar.gz and /dev/null differ diff --git a/cluster/images/obs-csi-plugin/install_obsfs.sh b/cluster/images/obs-csi-plugin/install_obsfs.sh deleted file mode 100644 index 1159f5dd..00000000 --- a/cluster/images/obs-csi-plugin/install_obsfs.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash - -checkLinuxOS() { - if grep -i -q "EulerOS" /etc/os-release; then - # EulerOS - return 5 - elif [ -f "/etc/redhat-release" ]; then - # CentOS or RHEL or EulerOS - if grep -i -q "CentOS" /etc/redhat-release; then - return 1 - elif grep -i -q "EulerOS" /etc/redhat-release; then - return 5 - fi - return 1 - elif [ -f "/etc/issue" ]; then - # Ubuntu or Debian or EulerOS - if grep -i -q "ubuntu" /etc/issue; then - return 2 - elif grep -i -q "debian" /etc/issue; then - return 3 - elif grep -i -q "Euler" /etc/os-release; then - return 5 - fi - elif [ -f "/etc/fedora-release" ]; then - # Fedora - return 4 - else - # unknow OS - return 0 - fi -} - -buildAndInstallObsfs() { - echo "[INFO] Install dependencies for OBSFS" - - if which yum &>/dev/null; then - echo "[INFO] yum is available, try installing obsfs with yum" - yum install -y gcc libstdc++-devel gcc-c++ fuse fuse-devel curl-devel libxml2-devel mailcap git automake make - yum install -y openssl-devel - elif which apt-get &>/dev/null; then - echo "[INFO] apt-get is available, try installing obsfs with apt-get" - apt-get install -y build-essential git libfuse-dev libcurl4-openssl-dev libxml2-dev mime-support automake libtool - apt-get install -y pkg-config libssl-dev - else - echo "[ERROR] unsupported systems, please install obsfs manually" - fi - - echo "[INFO] Unzip the code files" - tar -xvf /var/lib/csi/huaweicloud-obs-obsfs.tar.gz -C /var/lib/csi/ - cd /var/lib/csi/huaweicloud-obs-obsfs - - echo "[INFO] Compile OBSFS" - bash build.sh - echo "[INFO] Install OBSFS" - bash install_obsfs.sh - - echo "[INFO] Check version and check if it works" - obsfs --version -} - -installObsfs(){ - echo "[INFO] Install obsfs from ${1}" - fileName=${1} - rm -rf /var/lib/csi/${fileName} - tar -zxvf /var/lib/csi/$fileName.tar.gz -C /var/lib/csi/ - - cd /var/lib/csi/${fileName} - bash install_obsfs.sh - - echo "[INFO] Check version and check if it works" - obsfs --version - - n=`obsfs --version | grep "Storage Service File System" | wc -l` - if [ "$n" = "0" ]; then - echo "[WARN] Package doesn't work, try using compile and install" - buildAndInstallObsfs - fi -} - -checkLinuxOS -osCode=$? -echo "[INFO] OS Code $osCode" - -if [ "$osCode" = 1 ]; then - echo "[INFO] OS is CentOS or EulerOS, use the package to install obsfs" - yum install -y openssl-devel fuse fuse-devel - installObsfs obsfs_CentOS7.6_amd64 -elif [ "$osCode" = 2 ]; then - echo "[INFO] OS is Ubuntu, use the package to install obsfs" - apt-get install -y libfuse-dev libcurl4-openssl-dev - installObsfs obsfs_Ubuntu16.04_amd64 -elif [ "$osCode" = 5 ]; then - buildAndInstallObsfs -else - buildAndInstallObsfs -fi diff --git a/cluster/images/obs-csi-plugin/install_s3fs.sh b/cluster/images/obs-csi-plugin/install_s3fs.sh new file mode 100644 index 00000000..560cba82 --- /dev/null +++ b/cluster/images/obs-csi-plugin/install_s3fs.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +checkS3fsInstalled() { + echo "[INFO] Check version and check if it works" + isInstalled=1 + command -v s3fs >/dev/null 2>&1 || { isInstalled=0; } + if [ ${isInstalled} = 0 ]; then + echo "[WARN] has no command named s3fs" + return ${isInstalled} + fi + echo "[INFO] s3fs has been installed" + return ${isInstalled} +} + +checkLinuxOS() { + if grep -i -q "EulerOS" /etc/os-release; then + # EulerOS + return 5 + elif [ -f "/etc/redhat-release" ]; then + # CentOS or RHEL or EulerOS + if grep -i -q "CentOS" /etc/redhat-release; then + return 1 + elif grep -i -q "Fedora" /etc/redhat-release; then + return 4 + elif grep -i -q "EulerOS" /etc/redhat-release; then + return 5 + fi + elif [ -f "/etc/issue" ]; then + # Ubuntu or Debian or Euler + if grep -i -q "ubuntu" /etc/issue; then + return 2 + elif grep -i -q "debian" /etc/issue; then + return 3 + elif grep -i -q "Euler" /etc/os-release; then + return 5 + fi + elif [ -f "/etc/fedora-release" ]; then + # Fedora + if grep -i -q "Fedora" /etc/fedora-release; then + return 4 + fi + else + # unknown OS + return 0 + fi +} + +installS3fs() { + echo "[INFO] Install dependencies and s3fs" + pgkManageCmd=${1} + + if which yum >/dev/null 2>&1 && [ ${pgkManageCmd} = "yum" ]; then + echo "[INFO] command 'yum' is available, trying to install s3fs with yum" + sudo yum -y install epel-release + sudo yum -y install s3fs-fuse + return + elif which apt >/dev/null 2>&1 && [ ${pgkManageCmd} = "apt" ]; then + echo "[INFO] command 'apt' is available, trying to install s3fs with apt" + sudo apt -y install s3fs + return + elif which dnf >/dev/null 2>&1 && [ ${pgkManageCmd} = "dnf" ]; then + echo "[INFO] command 'dnf' is available, trying to install s3fs with dnf" + sudo dnf -y install s3fs-fuse + return + else + echo "[ERROR] unsupported systems, please install s3fs manually" + fi +} + +# pre install +checkS3fsInstalled +isInstalled=$? +if [ ${isInstalled} = 1 ]; then + exit 1 +fi + +# install +checkLinuxOS +osCode=$? +echo "[INFO] OS Code is ${osCode}" + +if [ ${osCode} = 1 ] || [ ${osCode} = 5 ]; then + echo "[INFO] OS is CentOS or EulerOS, use yum to install s3fs" + installS3fs "yum" +elif [ ${osCode} = 2 ] || [ ${osCode} = 3 ]; then + echo "[INFO] OS is Ubuntu or Debian, use apt to install s3fs" + installS3fs "apt" +elif [ ${osCode} = 4 ]; then + echo "[INFO] OS is Fedora, use dnf to install s3fs" + installS3fs "dnf" +fi + +# post install +checkS3fsInstalled +isInstalled=$? +if [ ${isInstalled} = 1 ]; then + exit 1 +fi +echo "[ERROR] failed to install s3fs, please install s3fs manually" diff --git a/cluster/images/obs-csi-plugin/obsfs_CentOS7.6_amd64.tar.gz b/cluster/images/obs-csi-plugin/obsfs_CentOS7.6_amd64.tar.gz deleted file mode 100644 index f4fde631..00000000 Binary files a/cluster/images/obs-csi-plugin/obsfs_CentOS7.6_amd64.tar.gz and /dev/null differ diff --git a/cluster/images/obs-csi-plugin/obsfs_Ubuntu16.04_amd64.tar.gz b/cluster/images/obs-csi-plugin/obsfs_Ubuntu16.04_amd64.tar.gz deleted file mode 100644 index 83e5310a..00000000 Binary files a/cluster/images/obs-csi-plugin/obsfs_Ubuntu16.04_amd64.tar.gz and /dev/null differ diff --git a/pkg/obs/controllerserver.go b/pkg/obs/controllerserver.go index 4098d827..70606d88 100644 --- a/pkg/obs/controllerserver.go +++ b/pkg/obs/controllerserver.go @@ -43,7 +43,7 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum return nil, err } - if volume, err := services.GetParallelFSBucket(credentials, volName); err != nil && status.Code(err) != codes.NotFound { + if volume, err := services.GetObsBucket(credentials, volName); err != nil && status.Code(err) != codes.NotFound { return nil, err } else if volume != nil { log.Infof("Volume %s existence, skip creating", volName) @@ -61,7 +61,7 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum } } - volume, err := services.GetParallelFSBucket(credentials, volName) + volume, err := services.GetObsBucket(credentials, volName) if err != nil { return nil, err } @@ -80,7 +80,7 @@ func (cs *controllerServer) DeleteVolume(_ context.Context, req *csi.DeleteVolum } credentials := cs.Driver.cloud - volume, err := services.GetParallelFSBucket(credentials, volName) + volume, err := services.GetObsBucket(credentials, volName) if err != nil { if common.IsNotFound(err) { log.Infof("Volume %s does not exist, skip deleting", volName) @@ -115,7 +115,7 @@ func (cs *controllerServer) ControllerGetVolume(_ context.Context, req *csi.Cont return nil, status.Error(codes.InvalidArgument, "Validation failed, volume ID cannot be empty") } - bucket, err := services.GetParallelFSBucket(cs.Driver.cloud, volumeID) + bucket, err := services.GetObsBucket(cs.Driver.cloud, volumeID) if err != nil { return nil, err } @@ -216,7 +216,7 @@ func (cs *controllerServer) ValidateVolumeCapabilities(_ context.Context, req *c if len(volumeID) == 0 { return nil, status.Error(codes.InvalidArgument, "Validation failed, volume ID cannot be empty") } - if _, err := services.GetParallelFSBucket(cs.Driver.cloud, volumeID); err != nil { + if _, err := services.GetObsBucket(cs.Driver.cloud, volumeID); err != nil { return nil, err } @@ -263,7 +263,7 @@ func (cs *controllerServer) ControllerExpandVolume(_ context.Context, req *csi.C "Validation failed, after round-up volume size %v exceeds the max size %v", sizeBytes, maxSizeBytes) } - volume, err := services.GetParallelFSBucket(cc, volumeID) + volume, err := services.GetObsBucket(cc, volumeID) if err != nil { return nil, err } diff --git a/pkg/obs/mount.go b/pkg/obs/mount.go index 5322cade..ac3205cb 100644 --- a/pkg/obs/mount.go +++ b/pkg/obs/mount.go @@ -26,7 +26,6 @@ func sendCommand(cmd CommandRPC, mountClient http.Client) error { if err != nil { return err } - log.Infof("Start sending command: %s", string(marshal)) response, err := mountClient.Post("http://unix", "application/json", bytes.NewReader(marshal)) if err != nil { return status.Errorf(codes.Internal, "Failed to post command, err: %v", err) @@ -34,6 +33,7 @@ func sendCommand(cmd CommandRPC, mountClient http.Client) error { defer response.Body.Close() if response.StatusCode != http.StatusOK { respBody, err := ioutil.ReadAll(response.Body) + log.Infof("start to mount bucket, cmd: %v", string(respBody)) if err != nil { return status.Errorf(codes.Internal, "Failed to read responseBody, err: %v", err) } diff --git a/pkg/obs/nodeserver.go b/pkg/obs/nodeserver.go index bd2ef204..817c1a51 100644 --- a/pkg/obs/nodeserver.go +++ b/pkg/obs/nodeserver.go @@ -47,9 +47,10 @@ type nodeServer struct { } const ( - credentialDir = "/var/lib/csi" - SocketPath = "/var/lib/csi/connector.sock" - perm = 0600 + credentialDir = "/var/lib/csi" + credentialFileName = ".passwd-s3fs" + SocketPath = "/var/lib/csi/connector.sock" + perm = 0600 ) func (ns *nodeServer) NodeStageVolume(_ context.Context, req *csi.NodeStageVolumeRequest) ( @@ -75,7 +76,7 @@ func (ns *nodeServer) NodePublishVolume(_ context.Context, req *csi.NodePublishV } credentials := ns.Driver.cloud - volume, err := services.GetParallelFSBucket(credentials, volumeID) + volume, err := services.GetObsBucket(credentials, volumeID) if err != nil { return nil, err } @@ -93,7 +94,7 @@ func (ns *nodeServer) NodePublishVolume(_ context.Context, req *csi.NodePublishV return nil, status.Errorf(codes.Internal, "Failed to make dir: %s, error: %v", targetPath, err) } - credentialFile := fmt.Sprintf("%s/%s", credentialDir, uuid.New().String()) + credentialFile := filepath.Join(credentialDir, uuid.New().String(), credentialFileName) accessKey := ns.Driver.cloud.Global.AccessKey secretKey := ns.Driver.cloud.Global.SecretKey if err := createCredentialFile(accessKey, secretKey, credentialFile); err != nil { @@ -101,14 +102,9 @@ func (ns *nodeServer) NodePublishVolume(_ context.Context, req *csi.NodePublishV } defer deleteCredentialFile(credentialFile) - mountFlags := []string{"big_writes", "max_write=131072", "use_ino"} + mountFlags := []string{"big_writes", "max_write=131072", "nonempty"} if mnt := req.GetVolumeCapability().GetMount(); mnt != nil { - for _, v := range mnt.GetMountFlags() { - if v == "passwd_file" || v == "use_ino" { - continue - } - mountFlags = append(mountFlags, v) - } + mountFlags = append(mountFlags, mnt.GetMountFlags()...) } parameters := map[string]string{ @@ -238,7 +234,7 @@ func (ns *nodeServer) NodeGetVolumeStats(_ context.Context, req *csi.NodeGetVolu log.Infof("NodeGetVolumeStats: stats info :%s", protosanitizer.StripSecrets(*stats)) capacity, usedBytes := stats.TotalBytes, stats.UsedBytes - bucket, err := services.GetParallelFSBucket(ns.Driver.cloud, volumeID) + bucket, err := services.GetObsBucket(ns.Driver.cloud, volumeID) if err != nil { return nil, err } diff --git a/pkg/obs/services/buckets.go b/pkg/obs/services/buckets.go index 0bce1062..863fe417 100644 --- a/pkg/obs/services/buckets.go +++ b/pkg/obs/services/buckets.go @@ -36,13 +36,13 @@ type Bucket struct { Capacity int64 } -func GetParallelFSBucket(c *config.CloudCredentials, bucketName string) (*Bucket, error) { +func GetObsBucket(c *config.CloudCredentials, bucketName string) (*Bucket, error) { metadata, err := GetBucketMetadata(c, bucketName) if err != nil { return nil, err } - if isParallelFile := IsParallelFSBucket(metadata.FSStatus); !isParallelFile { - return nil, status.Errorf(codes.Unavailable, "Error, the OBS instance %s is not a parallel file system", bucketName) + if ok := isObsBucket(metadata.FSStatus); !ok { + return nil, status.Errorf(codes.Unavailable, "Error, the OBS instance %s is not a obs bucket", bucketName) } capacity, err := GetBucketCapacity(c, bucketName) if err != nil { @@ -82,8 +82,8 @@ func CheckBucketExist(c *config.CloudCredentials, bucketName string) (bool, erro return true, nil } -func IsParallelFSBucket(FSStatus obs.FSStatusType) bool { - return FSStatus == obs.FSStatusEnabled +func isObsBucket(FSStatus obs.FSStatusType) bool { + return FSStatus == obs.FSStatusDisabled } func CreateBucket(c *config.CloudCredentials, bucketName string, acl obs.AclType) error { @@ -92,10 +92,9 @@ func CreateBucket(c *config.CloudCredentials, bucketName string, acl obs.AclType return err } input := &obs.CreateBucketInput{ - Bucket: bucketName, - ACL: acl, - IsFSFileInterface: true, - BucketLocation: obs.BucketLocation{Location: c.Global.Region}, + Bucket: bucketName, + ACL: acl, + BucketLocation: obs.BucketLocation{Location: c.Global.Region}, } if _, err = client.CreateBucket(input); err == nil { return nil @@ -215,7 +214,7 @@ func ListBuckets(c *config.CloudCredentials, opts ListOpts) ([]*Bucket, error) { } input := &obs.ListBucketsInput{ QueryLocation: false, - BucketType: obs.POSIX, + BucketType: obs.OBJECT, } output, err := client.ListBuckets(input) if err != nil { @@ -235,8 +234,8 @@ func ListBuckets(c *config.CloudCredentials, opts ListOpts) ([]*Bucket, error) { for k, j := 0, i+1; j <= i+opts.Limit && j < len(output.Buckets); k, j = k+1, j+1 { func(bucketName string, idx int) { group.Go(func() error { - fsBucket, err := GetParallelFSBucket(c, bucketName) - bucketList[idx] = fsBucket + obsBucket, err := GetObsBucket(c, bucketName) + bucketList[idx] = obsBucket return err }) }(output.Buckets[j].Name, k) diff --git a/pkg/utils/logroundtripper_test.go b/pkg/utils/logroundtripper_test.go index ed967a5d..13a23d3b 100644 --- a/pkg/utils/logroundtripper_test.go +++ b/pkg/utils/logroundtripper_test.go @@ -67,8 +67,8 @@ func TestRoundTrip(t *testing.T) { name: "test6", request: httpNewRequest("POST", "https://hub.docker.com/", nil, t), response: &http.Response{ - Status: "404 Not Found", - StatusCode: http.StatusNotFound, + Status: "405 Method Not Allowed", + StatusCode: http.StatusMethodNotAllowed, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, @@ -80,8 +80,8 @@ func TestRoundTrip(t *testing.T) { name: "test7", request: httpNewRequest("PUT", "https://hub.docker.com/", nil, t), response: &http.Response{ - Status: "404 Not Found", - StatusCode: http.StatusNotFound, + Status: "405 Method Not Allowed", + StatusCode: http.StatusMethodNotAllowed, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, @@ -93,8 +93,8 @@ func TestRoundTrip(t *testing.T) { name: "test8", request: httpNewRequest("DELETE", "https://hub.docker.com/", nil, t), response: &http.Response{ - Status: "404 Not Found", - StatusCode: http.StatusNotFound, + Status: "405 Method Not Allowed", + StatusCode: http.StatusMethodNotAllowed, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1,