Merge pull request #3 from seatable/seatable-plugins

Seatable plugins

docker/Dockerfile

@@ -1,5 +1,5 @@
 FROM phusion/baseimage:0.11
-# docker.seafile.top/seafile-dev/seatable-thumbnail-server:1.3.0
+# docker.seafile.top/seafile-dev/seatable-thumbnail-server:1.x.x
 
 # Aliyun ubuntu source
 RUN rm -rf /etc/apt/sources.list
@@ -15,7 +15,7 @@ RUN apt-get install -y nginx
 RUN apt-get install -y mysql-client
 
 # set python3.6 global
-RUN apt-get install -y python3 python3-pip python3-setuptools python3-ldap
+RUN apt-get install -y python3 python3-pip
 RUN python3.6 -m pip install --upgrade pip -i https://mirrors.aliyun.com/pypi/simple && rm -r /root/.cache/pip
 RUN rm -f /usr/bin/python && \
     rm -f /usr/bin/python3 && \
@@ -26,7 +26,7 @@ RUN rm -f /usr/bin/python && \
     ln -s /usr/local/bin/pip3.6 /usr/bin/pip && \
     ln -s /usr/local/bin/pip3.6 /usr/bin/pip3
 
-RUN pip3 install uvicorn pillow pymysql sqlalchemy future \
+RUN pip3 install uvicorn pillow pymysql sqlalchemy future requests \
     -i https://mirrors.aliyun.com/pypi/simple && rm -r /root/.cache/pip
 
 

docker/scripts/01_init.sh

@@ -28,7 +28,7 @@ if [[ ! -e /shared/seatable-thumbnail/ccnet ]]; then
     mkdir /shared/seatable-thumbnail/ccnet
 fi
 
-if [[ ! -e /shared/seatable-thumbnail/seafile-data ]]; then
+if [[ ! -e /shared/seatable-thumbnail/seafile-data && ! -e /opt/seatable-thumbnail/seafile-data ]]; then
     mkdir /shared/seatable-thumbnail/seafile-data
 fi
 

docker/scripts/init_config.py

@@ -60,6 +60,8 @@
 MYSQL_HOST = 'host'
 MYSQL_PORT = '3306'
 DATABASE_NAME = 'db_name'
+
+PLUGINS_REPO_ID = ''
 """
 
 if not os.path.exists(seatable_thumbnail_config_path):
@@ -95,6 +97,10 @@
     # location /thumbnail/ {
     #     proxy_pass https://thumbnail.seatable.cn/thumbnail/;
     # }
+    #
+    # location /dtable-plugins/ {
+    #     proxy_pass https://thumbnail.seatable.cn/dtable-plugins/;
+    # }
 
 }
 """

main.py

@@ -1,12 +1,14 @@
 import logging
 
 from seatable_thumbnail import DBSession
-from seatable_thumbnail.serializers import ThumbnailSerializer
+from seatable_thumbnail.serializers import ThumbnailSerializer, PluginSerializer
 from seatable_thumbnail.permissions import ThumbnailPermission
 from seatable_thumbnail.thumbnail import Thumbnail
+from seatable_thumbnail.plugin import Plugin
 from seatable_thumbnail.http_request import HTTPRequest
-from seatable_thumbnail.http_response import gen_error_response, \
+from seatable_thumbnail.http_response import gen_error_response, gen_plugin_response, \
     gen_text_response, gen_thumbnail_response, gen_cache_response
+from seatable_thumbnail.utils import cache_check
 
 logger = logging.getLogger(__name__)
 
@@ -73,16 +75,7 @@ async def __call__(self, scope, receive, send):
 
             # cache
             try:
-                etag = thumbnail_info.get('etag')
-                if_none_match_headers = request.headers.get('if-none-match')
-                if_none_match = if_none_match_headers[0] if if_none_match_headers else ''
-
-                last_modified = thumbnail_info.get('last_modified')
-                if_modified_since_headers = request.headers.get('if-modified-since')
-                if_modified_since = if_modified_since_headers[0] if if_modified_since_headers else ''
-
-                if (if_none_match and if_none_match == etag) \
-                        or (if_modified_since and if_modified_since == last_modified):
+                if cache_check(request, thumbnail_info):
                     response_start, response_body = gen_cache_response()
                     await send(response_start)
                     await send(response_body)
@@ -110,6 +103,56 @@ async def __call__(self, scope, receive, send):
                 await send(response_body)
                 return
 
+# ===== plugin =====
+        elif 'dtable-plugins/' == request.url[:15]:
+            db_session = DBSession()
+
+            # serializer
+            try:
+                serializer = PluginSerializer(db_session, request)
+                plugin_info = serializer.plugin_info
+            except Exception as e:
+                logger.exception(e)
+                db_session.close()
+                response_start, response_body = gen_error_response(
+                    400, 'Bad request.')
+                await send(response_start)
+                await send(response_body)
+                return
+
+            db_session.close()
+
+            # cache
+            try:
+                if cache_check(request, plugin_info):
+                    response_start, response_body = gen_cache_response()
+                    await send(response_start)
+                    await send(response_body)
+                    return
+            except Exception as e:
+                logger.exception(e)
+
+            # get
+            try:
+                plugin = Plugin(**plugin_info)
+                body = plugin.body
+                content_type = plugin.content_type
+                last_modified = plugin.last_modified
+                etag = plugin.etag
+
+                response_start, response_body = gen_plugin_response(
+                    body, content_type, etag, last_modified)
+                await send(response_start)
+                await send(response_body)
+                return
+            except Exception as e:
+                logger.exception(e)
+                response_start, response_body = gen_error_response(
+                    500, 'Internal server error.')
+                await send(response_start)
+                await send(response_body)
+                return
+
 # ===== Not found =====
         else:
             response_start, response_body = gen_error_response(

requirements.txt

@@ -3,6 +3,7 @@ pillow
 pymysql
 sqlalchemy
 future  # seaf-server
+requests
 
 # psd_tools
 # moviepy

seatable_thumbnail/http_response.py

@@ -51,3 +51,16 @@ def gen_thumbnail_response(thumbnail, etag, last_modified):
         response_start['headers'].append([b'Last-Modified', last_modified.encode('utf-8')])
 
     return response_start, response_body
+
+
+def gen_plugin_response(plugin, content_type, etag, last_modified):
+    response_start = gen_response_start(200, content_type.encode('utf-8'))
+    response_body = gen_response_body(plugin)
+
+    # cache
+    if plugin:
+        response_start['headers'].append([b'Cache-Control', b'max-age=86400, public'])
+        response_start['headers'].append([b'ETag', etag.encode('utf-8')])
+        response_start['headers'].append([b'Last-Modified', last_modified.encode('utf-8')])
+
+    return response_start, response_body

seatable_thumbnail/models.py

@@ -91,3 +91,12 @@ class DjangoSession(Base):
     session_key = Column(String(40), primary_key=True)
     session_data = Column(Text)
     expire_date = Column(DateTime)
+
+
+class DTableSystemPlugins(Base):
+    __tablename__ = 'dtable_system_plugin'
+    id = Column(Integer, primary_key=True)
+    added_by = Column(String(255))
+    added_time = Column(DateTime)
+    info = Column(Text)
+    name = Column(String(255), index=True)

seatable_thumbnail/plugin.py

@@ -0,0 +1,18 @@
+import requests
+
+import seatable_thumbnail.settings as settings
+from seatable_thumbnail.constants import EMPTY_BYTES
+from seatable_thumbnail.utils import get_inner_path
+
+
+class Plugin(object):
+    def __init__(self, **info):
+        self.__dict__.update(info)
+        self.body = EMPTY_BYTES
+        self.get()
+
+    def get(self):
+        inner_path = get_inner_path(
+            settings.PLUGINS_REPO_ID, self.file_id, self.file_name)
+        response = requests.get(inner_path)
+        self.body = response.content

seatable_thumbnail/serializers.py

@@ -2,13 +2,15 @@
 import uuid
 import json
 import base64
+import mimetypes
+from datetime import datetime
 from email.utils import formatdate
 
-from seaserv import seafile_api
 import seatable_thumbnail.settings as settings
-from seatable_thumbnail.constants import FILE_EXT_TYPE_MAP, \
+from seatable_thumbnail.constants import TEXT_CONTENT_TYPE, FILE_EXT_TYPE_MAP, \
     IMAGE, PSD, VIDEO, XMIND
-from seatable_thumbnail.models import Workspaces, DjangoSession
+from seatable_thumbnail.models import Workspaces, DjangoSession, DTableSystemPlugins
+from seatable_thumbnail.utils import get_file_id
 
 
 class ThumbnailSerializer(object):
@@ -22,7 +24,6 @@ def check(self):
         self.params_check()
         self.session_check()
         self.resource_check()
-        self.gen_thumbnail_info()
 
     def gen_thumbnail_info(self):
         thumbnail_info = {}
@@ -105,9 +106,7 @@ def resource_check(self):
             Workspaces).filter_by(id=workspace_id).first()
         repo_id = workspace.repo_id
         workspace_owner = workspace.owner
-        file_id = seafile_api.get_file_id_by_path(repo_id, file_path)
-        if not file_id:
-            raise ValueError(404, 'file_id not found.')
+        file_id = get_file_id(repo_id, file_path)
 
         thumbnail_dir = os.path.join(settings.THUMBNAIL_DIR, str(size))
         thumbnail_path = os.path.join(thumbnail_dir, file_id)
@@ -133,3 +132,61 @@ def exist_check(self, thumbnail_path):
             return True, last_modified
         else:
             return False, ''
+
+
+class PluginSerializer(object):
+    def __init__(self, db_session, request):
+        self.db_session = db_session
+        self.request = request
+        self.check()
+        self.gen_plugin_info()
+
+    def check(self):
+        self.params_check()
+        self.resource_check()
+
+    def gen_plugin_info(self):
+        plugin_info = {}
+        plugin_info.update(self.params)
+        plugin_info.update(self.resource)
+        self.plugin_info = plugin_info
+
+    def params_check(self):
+        plugin_name = self.request.url.split('/')[1]
+        path = self.request.query_dict['path'][0]
+        timestamp = self.request.query_dict['t'][0] if self.request.query_dict.get('t') else ''
+        version = self.request.query_dict['version'][0] if self.request.query_dict.get('version') else ''
+        content_type = mimetypes.guess_type(path)[0] if mimetypes.guess_type(path) else TEXT_CONTENT_TYPE.decode('utf-8')
+
+        self.params = {
+            'path': path,
+            'plugin_name': plugin_name,
+            'timestamp': timestamp,
+            'version': version,
+            'content_type': content_type,
+        }
+
+    def resource_check(self):
+        path = self.params['path']
+        plugin_name = self.params['plugin_name']
+
+        plugin = self.db_session.query(
+            DTableSystemPlugins).filter_by(name=plugin_name).first()
+
+        file_path ='/' + plugin.name + path
+        file_name = os.path.basename(file_path)
+        file_id = get_file_id(settings.PLUGINS_REPO_ID, file_path)
+
+        file_info = json.loads(plugin.info)
+        last_modified_time = datetime.strptime(file_info['last_modified'][:-6], '%Y-%m-%dT%H:%M:%S')
+        last_modified = formatdate(int(last_modified_time.timestamp()), usegmt=True)
+        etag = '"' + file_id + '"'
+
+        self.resource = {
+            'file_path': file_path,
+            'file_name': file_name,
+            'file_id': file_id,
+            'file_info': file_info,
+            'last_modified': last_modified,
+            'etag': etag,
+        }

seatable_thumbnail/settings.py

@@ -49,6 +49,10 @@
 THUMBNAIL_VIDEO_FRAME_TIME = 5
 
 
+# plugin
+PLUGINS_REPO_ID = ''
+
+
 # ======================== local settings ======================== #
 try:
     from local_settings import *

seatable_thumbnail/thumbnail.py

@@ -8,10 +8,11 @@
 from PIL import Image
 from email.utils import formatdate
 
-from seaserv import get_repo, get_file_size, seafile_api
+from seaserv import get_repo, get_file_size
 import seatable_thumbnail.settings as settings
 from seatable_thumbnail.constants import IMAGE_MODES, EMPTY_BYTES,\
     THUMBNAIL_EXTENSION, IMAGE, PSD, VIDEO, XMIND
+from seatable_thumbnail.utils import get_inner_path
 
 
 class Thumbnail(object):
@@ -36,8 +37,8 @@ def generate(self):
             if file_size > settings.THUMBNAIL_IMAGE_SIZE_LIMIT * 1024 * 1024:
                 raise AssertionError(400, 'file_size invalid.')
 
-            self.get_inner_path()
-            image_file = urllib.request.urlopen(self.inner_path)
+            inner_path = get_inner_path(self.repo_id, self.file_id, self.file_name)
+            image_file = urllib.request.urlopen(inner_path)
             f = BytesIO(image_file.read())
 
             image = Image.open(f)
@@ -48,8 +49,8 @@ def generate(self):
             from psd_tools import PSDImage
 
             tmp_psd = os.path.join(tempfile.gettempdir(), self.file_id)
-            self.get_inner_path()
-            urllib.request.urlretrieve(self.inner_path, tmp_psd)
+            inner_path = get_inner_path(self.repo_id, self.file_id, self.file_name)
+            urllib.request.urlretrieve(inner_path, tmp_psd)
 
             psd = PSDImage.open(tmp_psd)
             image = psd.topil()
@@ -63,8 +64,8 @@ def generate(self):
             tmp_image_path = os.path.join(
                 tempfile.gettempdir(), self.file_id + '.png')
             tmp_video = os.path.join(tempfile.gettempdir(), self.file_id)
-            self.get_inner_path()
-            urllib.request.urlretrieve(self.inner_path, tmp_video)
+            inner_path = get_inner_path(self.repo_id, self.file_id, self.file_name)
+            urllib.request.urlretrieve(inner_path, tmp_video)
 
             clip = VideoFileClip(tmp_video)
             clip.save_frame(
@@ -77,8 +78,8 @@ def generate(self):
 
 # ===== xmind =====
         elif self.file_type == XMIND:
-            self.get_inner_path()
-            xmind_file = urllib.request.urlopen(self.inner_path)
+            inner_path = get_inner_path(self.repo_id, self.file_id, self.file_name)
+            xmind_file = urllib.request.urlopen(inner_path)
             f = BytesIO(xmind_file.read())
             xmind_zip_file = zipfile.ZipFile(f, 'r')
             xmind_thumbnail = xmind_zip_file.read('Thumbnails/thumbnail.png')
@@ -87,14 +88,6 @@ def generate(self):
             image = Image.open(f)
             self.create_image_thumbnail(image)
 
-    def get_inner_path(self):
-        token = seafile_api.get_fileserver_access_token(
-            self.repo_id, self.file_id, 'view', '', use_onetime=True)
-        if not token:
-            raise ValueError(404, 'token not found.')
-        self.inner_path = '%s/files/%s/%s' % (
-            settings.INNER_FILE_SERVER_ROOT.rstrip('/'), token, urllib.parse.quote(self.file_name))
-
     def create_image_thumbnail(self, image):
         width, height = image.size
         image_memory_cost = width * height * 4 / 1024 / 1024