diff --git a/docker/Dockerfile b/docker/Dockerfile index 98bd5d8..da3cdbf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ FROM phusion/baseimage:0.11 -# docker.seafile.top/seafile-dev/seatable-thumbnail-server:1.3.0 +# docker.seafile.top/seafile-dev/seatable-thumbnail-server:1.x.x # Aliyun ubuntu source RUN rm -rf /etc/apt/sources.list @@ -15,7 +15,7 @@ RUN apt-get install -y nginx RUN apt-get install -y mysql-client # set python3.6 global -RUN apt-get install -y python3 python3-pip python3-setuptools python3-ldap +RUN apt-get install -y python3 python3-pip RUN python3.6 -m pip install --upgrade pip -i https://mirrors.aliyun.com/pypi/simple && rm -r /root/.cache/pip RUN rm -f /usr/bin/python && \ rm -f /usr/bin/python3 && \ @@ -26,7 +26,7 @@ RUN rm -f /usr/bin/python && \ ln -s /usr/local/bin/pip3.6 /usr/bin/pip && \ ln -s /usr/local/bin/pip3.6 /usr/bin/pip3 -RUN pip3 install uvicorn pillow pymysql sqlalchemy future \ +RUN pip3 install uvicorn pillow pymysql sqlalchemy future requests \ -i https://mirrors.aliyun.com/pypi/simple && rm -r /root/.cache/pip diff --git a/docker/scripts/01_init.sh b/docker/scripts/01_init.sh index ed9f563..6c1fd9f 100644 --- a/docker/scripts/01_init.sh +++ b/docker/scripts/01_init.sh @@ -28,7 +28,7 @@ if [[ ! -e /shared/seatable-thumbnail/ccnet ]]; then mkdir /shared/seatable-thumbnail/ccnet fi -if [[ ! -e /shared/seatable-thumbnail/seafile-data ]]; then +if [[ ! -e /shared/seatable-thumbnail/seafile-data && ! -e /opt/seatable-thumbnail/seafile-data ]]; then mkdir /shared/seatable-thumbnail/seafile-data fi diff --git a/docker/scripts/init_config.py b/docker/scripts/init_config.py index 904caf9..8b16376 100644 --- a/docker/scripts/init_config.py +++ b/docker/scripts/init_config.py @@ -60,6 +60,8 @@ MYSQL_HOST = 'host' MYSQL_PORT = '3306' DATABASE_NAME = 'db_name' + +PLUGINS_REPO_ID = '' """ if not os.path.exists(seatable_thumbnail_config_path): @@ -95,6 +97,10 @@ # location /thumbnail/ { # proxy_pass https://thumbnail.seatable.cn/thumbnail/; # } + # + # location /dtable-plugins/ { + # proxy_pass https://thumbnail.seatable.cn/dtable-plugins/; + # } } """ diff --git a/main.py b/main.py index af74cce..e4b541f 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,14 @@ import logging from seatable_thumbnail import DBSession -from seatable_thumbnail.serializers import ThumbnailSerializer +from seatable_thumbnail.serializers import ThumbnailSerializer, PluginSerializer from seatable_thumbnail.permissions import ThumbnailPermission from seatable_thumbnail.thumbnail import Thumbnail +from seatable_thumbnail.plugin import Plugin from seatable_thumbnail.http_request import HTTPRequest -from seatable_thumbnail.http_response import gen_error_response, \ +from seatable_thumbnail.http_response import gen_error_response, gen_plugin_response, \ gen_text_response, gen_thumbnail_response, gen_cache_response +from seatable_thumbnail.utils import cache_check logger = logging.getLogger(__name__) @@ -73,16 +75,7 @@ async def __call__(self, scope, receive, send): # cache try: - etag = thumbnail_info.get('etag') - if_none_match_headers = request.headers.get('if-none-match') - if_none_match = if_none_match_headers[0] if if_none_match_headers else '' - - last_modified = thumbnail_info.get('last_modified') - if_modified_since_headers = request.headers.get('if-modified-since') - if_modified_since = if_modified_since_headers[0] if if_modified_since_headers else '' - - if (if_none_match and if_none_match == etag) \ - or (if_modified_since and if_modified_since == last_modified): + if cache_check(request, thumbnail_info): response_start, response_body = gen_cache_response() await send(response_start) await send(response_body) @@ -110,6 +103,56 @@ async def __call__(self, scope, receive, send): await send(response_body) return +# ===== plugin ===== + elif 'dtable-plugins/' == request.url[:15]: + db_session = DBSession() + + # serializer + try: + serializer = PluginSerializer(db_session, request) + plugin_info = serializer.plugin_info + except Exception as e: + logger.exception(e) + db_session.close() + response_start, response_body = gen_error_response( + 400, 'Bad request.') + await send(response_start) + await send(response_body) + return + + db_session.close() + + # cache + try: + if cache_check(request, plugin_info): + response_start, response_body = gen_cache_response() + await send(response_start) + await send(response_body) + return + except Exception as e: + logger.exception(e) + + # get + try: + plugin = Plugin(**plugin_info) + body = plugin.body + content_type = plugin.content_type + last_modified = plugin.last_modified + etag = plugin.etag + + response_start, response_body = gen_plugin_response( + body, content_type, etag, last_modified) + await send(response_start) + await send(response_body) + return + except Exception as e: + logger.exception(e) + response_start, response_body = gen_error_response( + 500, 'Internal server error.') + await send(response_start) + await send(response_body) + return + # ===== Not found ===== else: response_start, response_body = gen_error_response( diff --git a/requirements.txt b/requirements.txt index 51b919b..ac9626d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ pillow pymysql sqlalchemy future # seaf-server +requests # psd_tools # moviepy diff --git a/seatable_thumbnail/http_response.py b/seatable_thumbnail/http_response.py index ef27546..d59ed76 100644 --- a/seatable_thumbnail/http_response.py +++ b/seatable_thumbnail/http_response.py @@ -51,3 +51,16 @@ def gen_thumbnail_response(thumbnail, etag, last_modified): response_start['headers'].append([b'Last-Modified', last_modified.encode('utf-8')]) return response_start, response_body + + +def gen_plugin_response(plugin, content_type, etag, last_modified): + response_start = gen_response_start(200, content_type.encode('utf-8')) + response_body = gen_response_body(plugin) + + # cache + if plugin: + response_start['headers'].append([b'Cache-Control', b'max-age=86400, public']) + response_start['headers'].append([b'ETag', etag.encode('utf-8')]) + response_start['headers'].append([b'Last-Modified', last_modified.encode('utf-8')]) + + return response_start, response_body diff --git a/seatable_thumbnail/models.py b/seatable_thumbnail/models.py index eb90fbb..2246ea3 100644 --- a/seatable_thumbnail/models.py +++ b/seatable_thumbnail/models.py @@ -91,3 +91,12 @@ class DjangoSession(Base): session_key = Column(String(40), primary_key=True) session_data = Column(Text) expire_date = Column(DateTime) + + +class DTableSystemPlugins(Base): + __tablename__ = 'dtable_system_plugin' + id = Column(Integer, primary_key=True) + added_by = Column(String(255)) + added_time = Column(DateTime) + info = Column(Text) + name = Column(String(255), index=True) diff --git a/seatable_thumbnail/plugin.py b/seatable_thumbnail/plugin.py new file mode 100644 index 0000000..b063a2c --- /dev/null +++ b/seatable_thumbnail/plugin.py @@ -0,0 +1,18 @@ +import requests + +import seatable_thumbnail.settings as settings +from seatable_thumbnail.constants import EMPTY_BYTES +from seatable_thumbnail.utils import get_inner_path + + +class Plugin(object): + def __init__(self, **info): + self.__dict__.update(info) + self.body = EMPTY_BYTES + self.get() + + def get(self): + inner_path = get_inner_path( + settings.PLUGINS_REPO_ID, self.file_id, self.file_name) + response = requests.get(inner_path) + self.body = response.content diff --git a/seatable_thumbnail/serializers.py b/seatable_thumbnail/serializers.py index 60d6b89..f5e8a66 100644 --- a/seatable_thumbnail/serializers.py +++ b/seatable_thumbnail/serializers.py @@ -2,13 +2,15 @@ import uuid import json import base64 +import mimetypes +from datetime import datetime from email.utils import formatdate -from seaserv import seafile_api import seatable_thumbnail.settings as settings -from seatable_thumbnail.constants import FILE_EXT_TYPE_MAP, \ +from seatable_thumbnail.constants import TEXT_CONTENT_TYPE, FILE_EXT_TYPE_MAP, \ IMAGE, PSD, VIDEO, XMIND -from seatable_thumbnail.models import Workspaces, DjangoSession +from seatable_thumbnail.models import Workspaces, DjangoSession, DTableSystemPlugins +from seatable_thumbnail.utils import get_file_id class ThumbnailSerializer(object): @@ -22,7 +24,6 @@ def check(self): self.params_check() self.session_check() self.resource_check() - self.gen_thumbnail_info() def gen_thumbnail_info(self): thumbnail_info = {} @@ -105,9 +106,7 @@ def resource_check(self): Workspaces).filter_by(id=workspace_id).first() repo_id = workspace.repo_id workspace_owner = workspace.owner - file_id = seafile_api.get_file_id_by_path(repo_id, file_path) - if not file_id: - raise ValueError(404, 'file_id not found.') + file_id = get_file_id(repo_id, file_path) thumbnail_dir = os.path.join(settings.THUMBNAIL_DIR, str(size)) thumbnail_path = os.path.join(thumbnail_dir, file_id) @@ -133,3 +132,61 @@ def exist_check(self, thumbnail_path): return True, last_modified else: return False, '' + + +class PluginSerializer(object): + def __init__(self, db_session, request): + self.db_session = db_session + self.request = request + self.check() + self.gen_plugin_info() + + def check(self): + self.params_check() + self.resource_check() + + def gen_plugin_info(self): + plugin_info = {} + plugin_info.update(self.params) + plugin_info.update(self.resource) + self.plugin_info = plugin_info + + def params_check(self): + plugin_name = self.request.url.split('/')[1] + path = self.request.query_dict['path'][0] + timestamp = self.request.query_dict['t'][0] if self.request.query_dict.get('t') else '' + version = self.request.query_dict['version'][0] if self.request.query_dict.get('version') else '' + content_type = mimetypes.guess_type(path)[0] if mimetypes.guess_type(path) else TEXT_CONTENT_TYPE.decode('utf-8') + + self.params = { + 'path': path, + 'plugin_name': plugin_name, + 'timestamp': timestamp, + 'version': version, + 'content_type': content_type, + } + + def resource_check(self): + path = self.params['path'] + plugin_name = self.params['plugin_name'] + + plugin = self.db_session.query( + DTableSystemPlugins).filter_by(name=plugin_name).first() + + file_path ='/' + plugin.name + path + file_name = os.path.basename(file_path) + file_id = get_file_id(settings.PLUGINS_REPO_ID, file_path) + + file_info = json.loads(plugin.info) + last_modified_time = datetime.strptime(file_info['last_modified'][:-6], '%Y-%m-%dT%H:%M:%S') + last_modified = formatdate(int(last_modified_time.timestamp()), usegmt=True) + etag = '"' + file_id + '"' + + self.resource = { + 'file_path': file_path, + 'file_name': file_name, + 'file_id': file_id, + 'file_info': file_info, + 'last_modified': last_modified, + 'etag': etag, + } \ No newline at end of file diff --git a/seatable_thumbnail/settings.py b/seatable_thumbnail/settings.py index 38fc381..44a5edc 100644 --- a/seatable_thumbnail/settings.py +++ b/seatable_thumbnail/settings.py @@ -49,6 +49,10 @@ THUMBNAIL_VIDEO_FRAME_TIME = 5 +# plugin +PLUGINS_REPO_ID = '' + + # ======================== local settings ======================== # try: from local_settings import * diff --git a/seatable_thumbnail/thumbnail.py b/seatable_thumbnail/thumbnail.py index 6889e42..d37584a 100644 --- a/seatable_thumbnail/thumbnail.py +++ b/seatable_thumbnail/thumbnail.py @@ -8,10 +8,11 @@ from PIL import Image from email.utils import formatdate -from seaserv import get_repo, get_file_size, seafile_api +from seaserv import get_repo, get_file_size import seatable_thumbnail.settings as settings from seatable_thumbnail.constants import IMAGE_MODES, EMPTY_BYTES,\ THUMBNAIL_EXTENSION, IMAGE, PSD, VIDEO, XMIND +from seatable_thumbnail.utils import get_inner_path class Thumbnail(object): @@ -36,8 +37,8 @@ def generate(self): if file_size > settings.THUMBNAIL_IMAGE_SIZE_LIMIT * 1024 * 1024: raise AssertionError(400, 'file_size invalid.') - self.get_inner_path() - image_file = urllib.request.urlopen(self.inner_path) + inner_path = get_inner_path(self.repo_id, self.file_id, self.file_name) + image_file = urllib.request.urlopen(inner_path) f = BytesIO(image_file.read()) image = Image.open(f) @@ -48,8 +49,8 @@ def generate(self): from psd_tools import PSDImage tmp_psd = os.path.join(tempfile.gettempdir(), self.file_id) - self.get_inner_path() - urllib.request.urlretrieve(self.inner_path, tmp_psd) + inner_path = get_inner_path(self.repo_id, self.file_id, self.file_name) + urllib.request.urlretrieve(inner_path, tmp_psd) psd = PSDImage.open(tmp_psd) image = psd.topil() @@ -63,8 +64,8 @@ def generate(self): tmp_image_path = os.path.join( tempfile.gettempdir(), self.file_id + '.png') tmp_video = os.path.join(tempfile.gettempdir(), self.file_id) - self.get_inner_path() - urllib.request.urlretrieve(self.inner_path, tmp_video) + inner_path = get_inner_path(self.repo_id, self.file_id, self.file_name) + urllib.request.urlretrieve(inner_path, tmp_video) clip = VideoFileClip(tmp_video) clip.save_frame( @@ -77,8 +78,8 @@ def generate(self): # ===== xmind ===== elif self.file_type == XMIND: - self.get_inner_path() - xmind_file = urllib.request.urlopen(self.inner_path) + inner_path = get_inner_path(self.repo_id, self.file_id, self.file_name) + xmind_file = urllib.request.urlopen(inner_path) f = BytesIO(xmind_file.read()) xmind_zip_file = zipfile.ZipFile(f, 'r') xmind_thumbnail = xmind_zip_file.read('Thumbnails/thumbnail.png') @@ -87,14 +88,6 @@ def generate(self): image = Image.open(f) self.create_image_thumbnail(image) - def get_inner_path(self): - token = seafile_api.get_fileserver_access_token( - self.repo_id, self.file_id, 'view', '', use_onetime=True) - if not token: - raise ValueError(404, 'token not found.') - self.inner_path = '%s/files/%s/%s' % ( - settings.INNER_FILE_SERVER_ROOT.rstrip('/'), token, urllib.parse.quote(self.file_name)) - def create_image_thumbnail(self, image): width, height = image.size image_memory_cost = width * height * 4 / 1024 / 1024 diff --git a/seatable_thumbnail/utils.py b/seatable_thumbnail/utils.py new file mode 100644 index 0000000..799ba1e --- /dev/null +++ b/seatable_thumbnail/utils.py @@ -0,0 +1,40 @@ +import urllib.request +import urllib.parse + +from seaserv import seafile_api +import seatable_thumbnail.settings as settings + + +def get_file_id(repo_id, file_path): + file_id = seafile_api.get_file_id_by_path(repo_id, file_path) + if not file_id: + raise ValueError(404, 'file_id not found.') + + return file_id + + +def get_inner_path(repo_id, file_id, file_name): + token = seafile_api.get_fileserver_access_token( + repo_id, file_id, 'view', '', use_onetime=True) + if not token: + raise ValueError(404, 'token not found.') + inner_path = '%s/files/%s/%s' % ( + settings.INNER_FILE_SERVER_ROOT.rstrip('/'), token, urllib.parse.quote(file_name)) + + return inner_path + + +def cache_check(request, info): + etag = info.get('etag') + if_none_match_headers = request.headers.get('if-none-match') + if_none_match = if_none_match_headers[0] if if_none_match_headers else '' + + last_modified = info.get('last_modified') + if_modified_since_headers = request.headers.get('if-modified-since') + if_modified_since = if_modified_since_headers[0] if if_modified_since_headers else '' + + if (if_none_match and if_none_match == etag) \ + or (if_modified_since and if_modified_since == last_modified): + return True + else: + return False