feat: Add plugin support for resource (dragonflyoss#291)

* feature: add plugin support for resource

Signed-off-by: Jim Ma <[email protected]>

* fix: skip link for mockgen import

Signed-off-by: Jim Ma <[email protected]>

* feature: add generate in Makefile

Signed-off-by: Jim Ma <[email protected]>

* fix: remove unused package

Signed-off-by: Jim Ma <[email protected]>

* fix: remove unused test

Signed-off-by: Jim Ma <[email protected]>

* feature: remove unused import package

Signed-off-by: Jim Ma <[email protected]>

Makefile

@@ -16,7 +16,7 @@ PROJECT_NAME := "d7y.io/dragonfly/v2"
 DFGET_NAME := "dfget"
 VERSION := "2.0.0"
 PKG := "$(PROJECT_NAME)"
-PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/ | grep -v '\(manager\)')
+PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/ | grep -v '\(/manager/\)')
 GIT_COMMIT := $(shell git rev-parse --verify HEAD --short=7)
 GIT_COMMIT_LONG := $(shell git rev-parse --verify HEAD)
 DFGET_ARCHIVE_PREFIX := "$(DFGET_NAME)_$(GIT_COMMIT)"
@@ -190,6 +190,11 @@ test-coverage:
  @cat cover.out >> coverage.txt
 .PHONY: test-coverage
 
+# Run go generate
+generate:
+ @go generate ${PKG_LIST}
+.PHONY: generate
+
 # Generate changelog
 changelog:
  @git-chglog -o CHANGELOG.md

internal/dfpath/dfpath.go

@@ -27,6 +27,7 @@ var (
  DaemonSockPath = filepath.Join(WorkHome, "daemon.sock")
  DaemonLockPath = filepath.Join(WorkHome, "daemon.lock")
  DfgetLockPath = filepath.Join(WorkHome, "dfget.lock")
+ PluginsDir = filepath.Join(WorkHome, "plugins")
 )
 
 func init() {

internal/dfplugin/dfplugin.go

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2020 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dfplugin
+
+import (
+ "errors"
+ "fmt"
+ "path"
+ "plugin"
+
+ "d7y.io/dragonfly/v2/internal/dfpath"
+)
+
+const (
+ // PluginFormat indicates the lookup name of a plugin in plugin directory.
+ PluginFormat = "d7y-%s-plugin-%s.so"
+
+ // PluginInitFuncName indicates the function `DragonflyPluginInit` must be implemented in plugin
+ PluginInitFuncName = "DragonflyPluginInit"
+
+ // PluginMetaKeyType indicates the type of a plugin, currently support: resource
+ PluginMetaKeyType = "type"
+ // PluginMetaKeyName indicates the name of a plugin
+ PluginMetaKeyName = "name"
+)
+
+type PluginType string
+
+const (
+ PluginTypeResource = PluginType("resource")
+)
+
+type PluginInitFunc func(option map[string]string) (plugin interface{}, meta map[string]string, err error)
+
+func Load(typ PluginType, name string, option map[string]string) (interface{}, map[string]string, error) {
+ soName := fmt.Sprintf(PluginFormat, string(typ), name)
+ p, err := plugin.Open(path.Join(dfpath.PluginsDir, soName))
+ if err != nil {
+ return nil, nil, err
+ }
+
+ symbol, err := p.Lookup(PluginInitFuncName)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ // FIXME when use symbol.(PluginInitFunc), ok is always false
+ f, ok := symbol.(func(option map[string]string) (plugin interface{}, meta map[string]string, err error))
+ if !ok {
+ return nil, nil, errors.New("invalid plugin init function signature")
+ }
+
+ i, meta, err := f(option)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ if meta == nil {
+ return nil, nil, errors.New("empty plugin metadata")
+ }
+
+ if meta[PluginMetaKeyType] != string(typ) {
+ return nil, nil, errors.New("plugin type not match")
+ }
+
+ if meta[PluginMetaKeyName] != name {
+ return nil, nil, errors.New("plugin name not match")
+ }
+ return i, meta, nil
+}

pkg/source/plugin.go

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2020 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package source
+
+import (
+ "errors"
+
+ "d7y.io/dragonfly/v2/internal/dfplugin"
+)
+
+const (
+ pluginMetadataSchema = "schema"
+)
+
+func LoadPlugin(schema string) (ResourceClient, error) {
+ // TODO init option
+ client, meta, err := dfplugin.Load(dfplugin.PluginTypeResource, schema, map[string]string{})
+ if err != nil {
+ return nil, err
+ }
+ if meta[pluginMetadataSchema] != schema {
+ return nil, errors.New("support schema not match")
+ }
+ if rc, ok := client.(ResourceClient); ok {
+ return rc, err
+ }
+ return nil, errors.New("invalid client, not a ResourceClient")
+}

pkg/source/plugin_test.go

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2020 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package source
+
+import (
+ "os"
+ "os/exec"
+ "path"
+ "testing"
+
+ testifyassert "github.com/stretchr/testify/assert"
+)
+
+func Test_loadPlugin(t *testing.T) {
+ assert := testifyassert.New(t)
+ defer func() {
+ os.Remove("./testdata/d7y-resource-plugin-dfs.so")
+ os.Remove("./testdata/test")
+ }()
+
+ var (
+ cmd *exec.Cmd
+ output []byte
+ wd string
+ err error
+ )
+
+ // TODO can not load golang plugin in testing, because the different building flags
+ // golang runtime will check the runtime hash of all imported packages
+
+ // build plugin
+ cmd = exec.Command("go", "build", "-buildmode=plugin", "-o=./testdata/d7y-resource-plugin-dfs.so", "testdata/plugin/dfs.go")
+ output, err = cmd.CombinedOutput()
+ assert.Nil(err)
+ if err != nil {
+ t.Fatalf(string(output))
+ return
+ }
+
+ // build test binary
+ cmd = exec.Command("go", "build", "-o=./testdata/test", "testdata/main.go")
+ output, err = cmd.CombinedOutput()
+ assert.Nil(err)
+ if err != nil {
+ t.Fatalf(string(output))
+ return
+ }
+
+ wd, err = os.Getwd()
+ assert.Nil(err)
+ wd = path.Join(wd, "testdata")
+
+ // execute test binary
+ cmd = exec.Command("./testdata/test", "-plugin-dir", wd)
+ output, err = cmd.CombinedOutput()
+ assert.Nil(err)
+ if err != nil {
+ t.Fatalf(string(output))
+ return
+ }
+}

pkg/source/source_client.go

@@ -23,6 +23,7 @@ import (
  "io"
  "net/url"
  "strings"
+ "sync"
  "time"
 
  logger "d7y.io/dragonfly/v2/internal/dflog"
@@ -62,6 +63,7 @@ type ClientManager interface {
 }
 
 type ClientManagerImpl struct {
+ sync.RWMutex
  clients map[string]ResourceClient
 }
 
@@ -197,9 +199,28 @@ func (clientMgr *ClientManagerImpl) getSourceClient(rawURL string) (ResourceClie
  if err != nil {
  return nil, err
  }
+ clientMgr.RLock()
  client, ok := clientMgr.clients[strings.ToLower(parsedURL.Scheme)]
+ clientMgr.RUnlock()
  if !ok || client == nil {
  return nil, fmt.Errorf("can not find client for supporting url %s, clients:%v", rawURL, clientMgr.clients)
  }
  return client, nil
 }
+
+func (clientMgr *ClientManagerImpl) loadSourcePlugin(schema string) (ResourceClient, error) {
+ clientMgr.Lock()
+ defer clientMgr.Unlock()
+ // double check
+ client, ok := clientMgr.clients[schema]
+ if ok {
+ return client, nil
+ }
+
+ client, err := LoadPlugin(schema)
+ if err != nil {
+ return nil, err
+ }
+ clientMgr.clients[schema] = client
+ return client, nil
+}

pkg/source/testdata/main.go

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2020 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "os"
+
+ "d7y.io/dragonfly/v2/internal/dfpath"
+ "d7y.io/dragonfly/v2/pkg/source"
+)
+
+func init() {
+ flag.StringVar(&dfpath.PluginsDir, "plugin-dir", ".", "")
+}
+
+func main() {
+ flag.Parse()
+
+ client, err := source.LoadPlugin("dfs")
+ if err != nil {
+ fmt.Printf("load plugin error: %s\n", err)
+ os.Exit(1)
+ }
+
+ ctx := context.Background()
+
+ l, err := client.GetContentLength(ctx, "", nil)
+ if err != nil {
+ fmt.Printf("get content length error: %s\n", err)
+ os.Exit(1)
+ }
+
+ rc, err := client.Download(ctx, "", nil)
+ if err != nil {
+ fmt.Printf("download error: %s\n", err)
+ os.Exit(1)
+ }
+
+ data, err := ioutil.ReadAll(rc)
+ if err != nil {
+ fmt.Printf("read error: %s\n", err)
+ os.Exit(1)
+ }
+
+ if l != int64(len(data)) {
+ fmt.Printf("content length mismatch\n")
+ os.Exit(1)
+ }
+
+ err = rc.Close()
+ if err != nil {
+ fmt.Printf("close error: %s\n", err)
+ os.Exit(1)
+ }
+}