From 5f05d427152726a19694afcc1c5dba68aa124d6a Mon Sep 17 00:00:00 2001 From: sniper-py Date: Tue, 22 Sep 2020 16:57:18 +0800 Subject: [PATCH 1/6] update seafile-data --- docker/scripts/01_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/scripts/01_init.sh b/docker/scripts/01_init.sh index ed9f563..6c1fd9f 100644 --- a/docker/scripts/01_init.sh +++ b/docker/scripts/01_init.sh @@ -28,7 +28,7 @@ if [[ ! -e /shared/seatable-thumbnail/ccnet ]]; then mkdir /shared/seatable-thumbnail/ccnet fi -if [[ ! -e /shared/seatable-thumbnail/seafile-data ]]; then +if [[ ! -e /shared/seatable-thumbnail/seafile-data && ! -e /opt/seatable-thumbnail/seafile-data ]]; then mkdir /shared/seatable-thumbnail/seafile-data fi From 82eab25932bae23c60c896b0b2f20306880bf30f Mon Sep 17 00:00:00 2001 From: sniper-py Date: Tue, 22 Sep 2020 16:57:44 +0800 Subject: [PATCH 2/6] plugins] model --- seatable_thumbnail/models.py | 9 +++++++++ seatable_thumbnail/settings.py | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/seatable_thumbnail/models.py b/seatable_thumbnail/models.py index eb90fbb..2246ea3 100644 --- a/seatable_thumbnail/models.py +++ b/seatable_thumbnail/models.py @@ -91,3 +91,12 @@ class DjangoSession(Base): session_key = Column(String(40), primary_key=True) session_data = Column(Text) expire_date = Column(DateTime) + + +class DTableSystemPlugins(Base): + __tablename__ = 'dtable_system_plugin' + id = Column(Integer, primary_key=True) + added_by = Column(String(255)) + added_time = Column(DateTime) + info = Column(Text) + name = Column(String(255), index=True) diff --git a/seatable_thumbnail/settings.py b/seatable_thumbnail/settings.py index 38fc381..44a5edc 100644 --- a/seatable_thumbnail/settings.py +++ b/seatable_thumbnail/settings.py @@ -49,6 +49,10 @@ THUMBNAIL_VIDEO_FRAME_TIME = 5 +# plugin +PLUGINS_REPO_ID = '' + + # ======================== local settings ======================== # try: from local_settings import * From 01ab1232835d41c074d7a6d58b9c84a415b9b0fc Mon Sep 17 00:00:00 2001 From: sniper-py Date: Tue, 22 Sep 2020 17:55:33 +0800 Subject: [PATCH 3/6] PluginSerializer --- main.py | 64 ++++++++++++++++++++++++++++- seatable_thumbnail/http_response.py | 13 ++++++ seatable_thumbnail/serializers.py | 61 ++++++++++++++++++++++++++- 3 files changed, 134 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index af74cce..d0d7bb5 100644 --- a/main.py +++ b/main.py @@ -1,11 +1,12 @@ import logging from seatable_thumbnail import DBSession -from seatable_thumbnail.serializers import ThumbnailSerializer +from seatable_thumbnail.serializers import ThumbnailSerializer, PluginSerializer from seatable_thumbnail.permissions import ThumbnailPermission from seatable_thumbnail.thumbnail import Thumbnail +from seatable_thumbnail.plugin import Plugin from seatable_thumbnail.http_request import HTTPRequest -from seatable_thumbnail.http_response import gen_error_response, \ +from seatable_thumbnail.http_response import gen_error_response, gen_plugin_response, \ gen_text_response, gen_thumbnail_response, gen_cache_response logger = logging.getLogger(__name__) @@ -110,6 +111,65 @@ async def __call__(self, scope, receive, send): await send(response_body) return +# ===== plugin ===== + elif 'dtable-plugins/' == request.url[:15]: + db_session = DBSession() + + # serializer + try: + serializer = PluginSerializer(db_session, request) + plugin_info = serializer.plugin_info + except Exception as e: + logger.exception(e) + db_session.close() + response_start, response_body = gen_error_response( + 400, 'Bad request.') + await send(response_start) + await send(response_body) + return + + db_session.close() + + # cache + try: + etag = plugin_info.get('etag') + if_none_match_headers = request.headers.get('if-none-match') + if_none_match = if_none_match_headers[0] if if_none_match_headers else '' + + last_modified = plugin_info.get('last_modified') + if_modified_since_headers = request.headers.get('if-modified-since') + if_modified_since = if_modified_since_headers[0] if if_modified_since_headers else '' + + if (if_none_match and if_none_match == etag) \ + or (if_modified_since and if_modified_since == last_modified): + response_start, response_body = gen_cache_response() + await send(response_start) + await send(response_body) + return + except Exception as e: + logger.exception(e) + + # get + try: + plugin = Plugin(**plugin_info) + body = plugin.body + content_type = plugin.content_type + last_modified = plugin.last_modified + etag = plugin.etag + + response_start, response_body = gen_plugin_response( + body, content_type, etag, last_modified) + await send(response_start) + await send(response_body) + return + except Exception as e: + logger.exception(e) + response_start, response_body = gen_error_response( + 500, 'Internal server error.') + await send(response_start) + await send(response_body) + return + # ===== Not found ===== else: response_start, response_body = gen_error_response( diff --git a/seatable_thumbnail/http_response.py b/seatable_thumbnail/http_response.py index ef27546..c026015 100644 --- a/seatable_thumbnail/http_response.py +++ b/seatable_thumbnail/http_response.py @@ -51,3 +51,16 @@ def gen_thumbnail_response(thumbnail, etag, last_modified): response_start['headers'].append([b'Last-Modified', last_modified.encode('utf-8')]) return response_start, response_body + + +def gen_plugin_response(plugin, content_type, etag, last_modified): + response_start = gen_response_start(200, content_type) + response_body = gen_response_body(plugin) + + # cache + if plugin: + response_start['headers'].append([b'Cache-Control', b'max-age=86400, public']) + response_start['headers'].append([b'ETag', etag.encode('utf-8')]) + response_start['headers'].append([b'Last-Modified', last_modified.encode('utf-8')]) + + return response_start, response_body diff --git a/seatable_thumbnail/serializers.py b/seatable_thumbnail/serializers.py index 60d6b89..0e5304a 100644 --- a/seatable_thumbnail/serializers.py +++ b/seatable_thumbnail/serializers.py @@ -2,13 +2,14 @@ import uuid import json import base64 +import mimetypes from email.utils import formatdate from seaserv import seafile_api import seatable_thumbnail.settings as settings from seatable_thumbnail.constants import FILE_EXT_TYPE_MAP, \ IMAGE, PSD, VIDEO, XMIND -from seatable_thumbnail.models import Workspaces, DjangoSession +from seatable_thumbnail.models import Workspaces, DjangoSession, DTableSystemPlugins class ThumbnailSerializer(object): @@ -22,7 +23,6 @@ def check(self): self.params_check() self.session_check() self.resource_check() - self.gen_thumbnail_info() def gen_thumbnail_info(self): thumbnail_info = {} @@ -133,3 +133,60 @@ def exist_check(self, thumbnail_path): return True, last_modified else: return False, '' + + +class PluginSerializer(object): + def __init__(self, db_session, request): + self.db_session = db_session + self.request = request + self.check() + self.gen_plugin_info() + + def check(self): + self.params_check() + self.resource_check() + + def gen_plugin_info(self): + plugin_info = {} + plugin_info.update(self.params) + plugin_info.update(self.resource) + self.plugin_info = plugin_info + + def params_check(self): + path = self.request.query_dict['path'][0] + plugin_name = self.request.url.split('/')[1] + timestamp = self.request.query_dict['t'][0] if self.request.query_dict.get('t') else '' + version = self.request.query_dict['version'][0] if self.request.query_dict.get('version') else '' + content_type = mimetypes.guess_type(path) + + self.params = { + 'path': path, + 'plugin_name': plugin_name, + 'timestamp': timestamp, + 'version': version, + 'content_type': content_type, + } + + def resource_check(self): + path = self.params['path'] + plugin_name = self.params['plugin_name'] + + plugin = self.db_session.query( + DTableSystemPlugins).filter_by(name=plugin_name).first() + + file_path ='/' + plugin.name + path + repo_id = settings.PLUGINS_REPO_ID + file_id = seafile_api.get_file_id_by_path(repo_id, file_path) + if not file_id: + raise ValueError(404, 'file_id not found.') + + info = json.loads(plugin.info) + last_modified = info['last_modified'] + etag = '"' + file_id + '"' + + self.resource = { + 'file_id': file_id, + 'info': info, + 'last_modified': last_modified, + 'etag': etag, + } \ No newline at end of file From c913a23fdb5c27074e0a4a300860cdf73592ad53 Mon Sep 17 00:00:00 2001 From: sniper-py Date: Tue, 22 Sep 2020 18:06:07 +0800 Subject: [PATCH 4/6] Plugin obj --- docker/scripts/init_config.py | 4 ++++ seatable_thumbnail/http_response.py | 2 +- seatable_thumbnail/plugin.py | 27 +++++++++++++++++++++++++++ seatable_thumbnail/serializers.py | 18 +++++++++++------- 4 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 seatable_thumbnail/plugin.py diff --git a/docker/scripts/init_config.py b/docker/scripts/init_config.py index 904caf9..6d7ee97 100644 --- a/docker/scripts/init_config.py +++ b/docker/scripts/init_config.py @@ -95,6 +95,10 @@ # location /thumbnail/ { # proxy_pass https://thumbnail.seatable.cn/thumbnail/; # } + # + # location /dtable-plugins/ { + # proxy_pass https://thumbnail.seatable.cn/dtable-plugins/; + # } } """ diff --git a/seatable_thumbnail/http_response.py b/seatable_thumbnail/http_response.py index c026015..d59ed76 100644 --- a/seatable_thumbnail/http_response.py +++ b/seatable_thumbnail/http_response.py @@ -54,7 +54,7 @@ def gen_thumbnail_response(thumbnail, etag, last_modified): def gen_plugin_response(plugin, content_type, etag, last_modified): - response_start = gen_response_start(200, content_type) + response_start = gen_response_start(200, content_type.encode('utf-8')) response_body = gen_response_body(plugin) # cache diff --git a/seatable_thumbnail/plugin.py b/seatable_thumbnail/plugin.py new file mode 100644 index 0000000..de74a03 --- /dev/null +++ b/seatable_thumbnail/plugin.py @@ -0,0 +1,27 @@ +import requests +import urllib.request +import urllib.parse + +from seaserv import seafile_api +import seatable_thumbnail.settings as settings +from seatable_thumbnail.constants import EMPTY_BYTES + + +class Plugin(object): + def __init__(self, **info): + self.__dict__.update(info) + self.body = EMPTY_BYTES + self.get() + + def get(self): + self.get_inner_path() + response = requests.get(self.inner_path) + self.body = response.content + + def get_inner_path(self): + token = seafile_api.get_fileserver_access_token( + settings.PLUGINS_REPO_ID, self.file_id, 'view', '', use_onetime=True) + if not token: + raise ValueError(404, 'token not found.') + self.inner_path = '%s/files/%s/%s' % ( + settings.INNER_FILE_SERVER_ROOT.rstrip('/'), token, urllib.parse.quote(self.file_name)) diff --git a/seatable_thumbnail/serializers.py b/seatable_thumbnail/serializers.py index 0e5304a..26cdc59 100644 --- a/seatable_thumbnail/serializers.py +++ b/seatable_thumbnail/serializers.py @@ -3,11 +3,12 @@ import json import base64 import mimetypes +from datetime import datetime from email.utils import formatdate from seaserv import seafile_api import seatable_thumbnail.settings as settings -from seatable_thumbnail.constants import FILE_EXT_TYPE_MAP, \ +from seatable_thumbnail.constants import TEXT_CONTENT_TYPE, FILE_EXT_TYPE_MAP, \ IMAGE, PSD, VIDEO, XMIND from seatable_thumbnail.models import Workspaces, DjangoSession, DTableSystemPlugins @@ -157,7 +158,7 @@ def params_check(self): plugin_name = self.request.url.split('/')[1] timestamp = self.request.query_dict['t'][0] if self.request.query_dict.get('t') else '' version = self.request.query_dict['version'][0] if self.request.query_dict.get('version') else '' - content_type = mimetypes.guess_type(path) + content_type = mimetypes.guess_type(path)[0] if mimetypes.guess_type(path) else TEXT_CONTENT_TYPE.decode('utf-8') self.params = { 'path': path, @@ -175,18 +176,21 @@ def resource_check(self): DTableSystemPlugins).filter_by(name=plugin_name).first() file_path ='/' + plugin.name + path - repo_id = settings.PLUGINS_REPO_ID - file_id = seafile_api.get_file_id_by_path(repo_id, file_path) + file_name = os.path.basename(file_path) + file_id = seafile_api.get_file_id_by_path(settings.PLUGINS_REPO_ID, file_path) if not file_id: raise ValueError(404, 'file_id not found.') - info = json.loads(plugin.info) - last_modified = info['last_modified'] + file_info = json.loads(plugin.info) + last_modified_time = datetime.strptime(file_info['last_modified'][:-6], '%Y-%m-%dT%H:%M:%S') + last_modified = formatdate(int(last_modified_time.timestamp()), usegmt=True) etag = '"' + file_id + '"' self.resource = { + 'file_path': file_path, + 'file_name': file_name, 'file_id': file_id, - 'info': info, + 'file_info': file_info, 'last_modified': last_modified, 'etag': etag, } \ No newline at end of file From 8c6ba5cfc4e83ec9c737e3c8987b59fe80f8b9fc Mon Sep 17 00:00:00 2001 From: sniper-py Date: Wed, 23 Sep 2020 11:55:39 +0800 Subject: [PATCH 5/6] get_file_id --- seatable_thumbnail/plugin.py | 6 ++++-- seatable_thumbnail/serializers.py | 28 +++++++++++++++++++++------- seatable_thumbnail/thumbnail.py | 18 ++++++++++-------- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/seatable_thumbnail/plugin.py b/seatable_thumbnail/plugin.py index de74a03..5a32400 100644 --- a/seatable_thumbnail/plugin.py +++ b/seatable_thumbnail/plugin.py @@ -14,8 +14,8 @@ def __init__(self, **info): self.get() def get(self): - self.get_inner_path() - response = requests.get(self.inner_path) + inner_path = self.get_inner_path() + response = requests.get(inner_path) self.body = response.content def get_inner_path(self): @@ -25,3 +25,5 @@ def get_inner_path(self): raise ValueError(404, 'token not found.') self.inner_path = '%s/files/%s/%s' % ( settings.INNER_FILE_SERVER_ROOT.rstrip('/'), token, urllib.parse.quote(self.file_name)) + + return self.inner_path diff --git a/seatable_thumbnail/serializers.py b/seatable_thumbnail/serializers.py index 26cdc59..00a955c 100644 --- a/seatable_thumbnail/serializers.py +++ b/seatable_thumbnail/serializers.py @@ -32,6 +32,13 @@ def gen_thumbnail_info(self): thumbnail_info.update(self.resource) self.thumbnail_info = thumbnail_info + def get_file_id(self): + self.file_id = seafile_api.get_file_id_by_path(self.repo_id, self.file_path) + if not self.file_id: + raise ValueError(404, 'file_id not found.') + + return self.file_id + def parse_django_session(self, session_data): # only for django 1.11.x encoded_data = base64.b64decode(session_data) @@ -106,9 +113,9 @@ def resource_check(self): Workspaces).filter_by(id=workspace_id).first() repo_id = workspace.repo_id workspace_owner = workspace.owner - file_id = seafile_api.get_file_id_by_path(repo_id, file_path) - if not file_id: - raise ValueError(404, 'file_id not found.') + self.repo_id = repo_id + self.file_path = file_path + file_id = self.get_file_id() thumbnail_dir = os.path.join(settings.THUMBNAIL_DIR, str(size)) thumbnail_path = os.path.join(thumbnail_dir, file_id) @@ -153,9 +160,16 @@ def gen_plugin_info(self): plugin_info.update(self.resource) self.plugin_info = plugin_info + def get_file_id(self): + self.file_id = seafile_api.get_file_id_by_path(self.repo_id, self.file_path) + if not self.file_id: + raise ValueError(404, 'file_id not found.') + + return self.file_id + def params_check(self): - path = self.request.query_dict['path'][0] plugin_name = self.request.url.split('/')[1] + path = self.request.query_dict['path'][0] timestamp = self.request.query_dict['t'][0] if self.request.query_dict.get('t') else '' version = self.request.query_dict['version'][0] if self.request.query_dict.get('version') else '' content_type = mimetypes.guess_type(path)[0] if mimetypes.guess_type(path) else TEXT_CONTENT_TYPE.decode('utf-8') @@ -177,9 +191,9 @@ def resource_check(self): file_path ='/' + plugin.name + path file_name = os.path.basename(file_path) - file_id = seafile_api.get_file_id_by_path(settings.PLUGINS_REPO_ID, file_path) - if not file_id: - raise ValueError(404, 'file_id not found.') + self.repo_id = settings.PLUGINS_REPO_ID + self.file_path = file_path + file_id = self.get_file_id() file_info = json.loads(plugin.info) last_modified_time = datetime.strptime(file_info['last_modified'][:-6], '%Y-%m-%dT%H:%M:%S') diff --git a/seatable_thumbnail/thumbnail.py b/seatable_thumbnail/thumbnail.py index 6889e42..002f067 100644 --- a/seatable_thumbnail/thumbnail.py +++ b/seatable_thumbnail/thumbnail.py @@ -36,8 +36,8 @@ def generate(self): if file_size > settings.THUMBNAIL_IMAGE_SIZE_LIMIT * 1024 * 1024: raise AssertionError(400, 'file_size invalid.') - self.get_inner_path() - image_file = urllib.request.urlopen(self.inner_path) + inner_path = self.get_inner_path() + image_file = urllib.request.urlopen(inner_path) f = BytesIO(image_file.read()) image = Image.open(f) @@ -48,8 +48,8 @@ def generate(self): from psd_tools import PSDImage tmp_psd = os.path.join(tempfile.gettempdir(), self.file_id) - self.get_inner_path() - urllib.request.urlretrieve(self.inner_path, tmp_psd) + inner_path = self.get_inner_path() + urllib.request.urlretrieve(inner_path, tmp_psd) psd = PSDImage.open(tmp_psd) image = psd.topil() @@ -63,8 +63,8 @@ def generate(self): tmp_image_path = os.path.join( tempfile.gettempdir(), self.file_id + '.png') tmp_video = os.path.join(tempfile.gettempdir(), self.file_id) - self.get_inner_path() - urllib.request.urlretrieve(self.inner_path, tmp_video) + inner_path = self.get_inner_path() + urllib.request.urlretrieve(inner_path, tmp_video) clip = VideoFileClip(tmp_video) clip.save_frame( @@ -77,8 +77,8 @@ def generate(self): # ===== xmind ===== elif self.file_type == XMIND: - self.get_inner_path() - xmind_file = urllib.request.urlopen(self.inner_path) + inner_path = self.get_inner_path() + xmind_file = urllib.request.urlopen(inner_path) f = BytesIO(xmind_file.read()) xmind_zip_file = zipfile.ZipFile(f, 'r') xmind_thumbnail = xmind_zip_file.read('Thumbnails/thumbnail.png') @@ -95,6 +95,8 @@ def get_inner_path(self): self.inner_path = '%s/files/%s/%s' % ( settings.INNER_FILE_SERVER_ROOT.rstrip('/'), token, urllib.parse.quote(self.file_name)) + return self.inner_path + def create_image_thumbnail(self, image): width, height = image.size image_memory_cost = width * height * 4 / 1024 / 1024 From 77d926759bf8e624964f96c4d2b11f094273d083 Mon Sep 17 00:00:00 2001 From: sniper-py Date: Wed, 23 Sep 2020 15:05:02 +0800 Subject: [PATCH 6/6] add utils --- docker/Dockerfile | 6 ++--- docker/scripts/init_config.py | 2 ++ main.py | 23 +++--------------- requirements.txt | 1 + seatable_thumbnail/plugin.py | 17 +++---------- seatable_thumbnail/serializers.py | 24 +++---------------- seatable_thumbnail/thumbnail.py | 21 +++++----------- seatable_thumbnail/utils.py | 40 +++++++++++++++++++++++++++++++ 8 files changed, 61 insertions(+), 73 deletions(-) create mode 100644 seatable_thumbnail/utils.py diff --git a/docker/Dockerfile b/docker/Dockerfile index 98bd5d8..da3cdbf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ FROM phusion/baseimage:0.11 -# docker.seafile.top/seafile-dev/seatable-thumbnail-server:1.3.0 +# docker.seafile.top/seafile-dev/seatable-thumbnail-server:1.x.x # Aliyun ubuntu source RUN rm -rf /etc/apt/sources.list @@ -15,7 +15,7 @@ RUN apt-get install -y nginx RUN apt-get install -y mysql-client # set python3.6 global -RUN apt-get install -y python3 python3-pip python3-setuptools python3-ldap +RUN apt-get install -y python3 python3-pip RUN python3.6 -m pip install --upgrade pip -i https://mirrors.aliyun.com/pypi/simple && rm -r /root/.cache/pip RUN rm -f /usr/bin/python && \ rm -f /usr/bin/python3 && \ @@ -26,7 +26,7 @@ RUN rm -f /usr/bin/python && \ ln -s /usr/local/bin/pip3.6 /usr/bin/pip && \ ln -s /usr/local/bin/pip3.6 /usr/bin/pip3 -RUN pip3 install uvicorn pillow pymysql sqlalchemy future \ +RUN pip3 install uvicorn pillow pymysql sqlalchemy future requests \ -i https://mirrors.aliyun.com/pypi/simple && rm -r /root/.cache/pip diff --git a/docker/scripts/init_config.py b/docker/scripts/init_config.py index 6d7ee97..8b16376 100644 --- a/docker/scripts/init_config.py +++ b/docker/scripts/init_config.py @@ -60,6 +60,8 @@ MYSQL_HOST = 'host' MYSQL_PORT = '3306' DATABASE_NAME = 'db_name' + +PLUGINS_REPO_ID = '' """ if not os.path.exists(seatable_thumbnail_config_path): diff --git a/main.py b/main.py index d0d7bb5..e4b541f 100644 --- a/main.py +++ b/main.py @@ -8,6 +8,7 @@ from seatable_thumbnail.http_request import HTTPRequest from seatable_thumbnail.http_response import gen_error_response, gen_plugin_response, \ gen_text_response, gen_thumbnail_response, gen_cache_response +from seatable_thumbnail.utils import cache_check logger = logging.getLogger(__name__) @@ -74,16 +75,7 @@ async def __call__(self, scope, receive, send): # cache try: - etag = thumbnail_info.get('etag') - if_none_match_headers = request.headers.get('if-none-match') - if_none_match = if_none_match_headers[0] if if_none_match_headers else '' - - last_modified = thumbnail_info.get('last_modified') - if_modified_since_headers = request.headers.get('if-modified-since') - if_modified_since = if_modified_since_headers[0] if if_modified_since_headers else '' - - if (if_none_match and if_none_match == etag) \ - or (if_modified_since and if_modified_since == last_modified): + if cache_check(request, thumbnail_info): response_start, response_body = gen_cache_response() await send(response_start) await send(response_body) @@ -132,16 +124,7 @@ async def __call__(self, scope, receive, send): # cache try: - etag = plugin_info.get('etag') - if_none_match_headers = request.headers.get('if-none-match') - if_none_match = if_none_match_headers[0] if if_none_match_headers else '' - - last_modified = plugin_info.get('last_modified') - if_modified_since_headers = request.headers.get('if-modified-since') - if_modified_since = if_modified_since_headers[0] if if_modified_since_headers else '' - - if (if_none_match and if_none_match == etag) \ - or (if_modified_since and if_modified_since == last_modified): + if cache_check(request, plugin_info): response_start, response_body = gen_cache_response() await send(response_start) await send(response_body) diff --git a/requirements.txt b/requirements.txt index 51b919b..ac9626d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ pillow pymysql sqlalchemy future # seaf-server +requests # psd_tools # moviepy diff --git a/seatable_thumbnail/plugin.py b/seatable_thumbnail/plugin.py index 5a32400..b063a2c 100644 --- a/seatable_thumbnail/plugin.py +++ b/seatable_thumbnail/plugin.py @@ -1,10 +1,8 @@ import requests -import urllib.request -import urllib.parse -from seaserv import seafile_api import seatable_thumbnail.settings as settings from seatable_thumbnail.constants import EMPTY_BYTES +from seatable_thumbnail.utils import get_inner_path class Plugin(object): @@ -14,16 +12,7 @@ def __init__(self, **info): self.get() def get(self): - inner_path = self.get_inner_path() + inner_path = get_inner_path( + settings.PLUGINS_REPO_ID, self.file_id, self.file_name) response = requests.get(inner_path) self.body = response.content - - def get_inner_path(self): - token = seafile_api.get_fileserver_access_token( - settings.PLUGINS_REPO_ID, self.file_id, 'view', '', use_onetime=True) - if not token: - raise ValueError(404, 'token not found.') - self.inner_path = '%s/files/%s/%s' % ( - settings.INNER_FILE_SERVER_ROOT.rstrip('/'), token, urllib.parse.quote(self.file_name)) - - return self.inner_path diff --git a/seatable_thumbnail/serializers.py b/seatable_thumbnail/serializers.py index 00a955c..f5e8a66 100644 --- a/seatable_thumbnail/serializers.py +++ b/seatable_thumbnail/serializers.py @@ -6,11 +6,11 @@ from datetime import datetime from email.utils import formatdate -from seaserv import seafile_api import seatable_thumbnail.settings as settings from seatable_thumbnail.constants import TEXT_CONTENT_TYPE, FILE_EXT_TYPE_MAP, \ IMAGE, PSD, VIDEO, XMIND from seatable_thumbnail.models import Workspaces, DjangoSession, DTableSystemPlugins +from seatable_thumbnail.utils import get_file_id class ThumbnailSerializer(object): @@ -32,13 +32,6 @@ def gen_thumbnail_info(self): thumbnail_info.update(self.resource) self.thumbnail_info = thumbnail_info - def get_file_id(self): - self.file_id = seafile_api.get_file_id_by_path(self.repo_id, self.file_path) - if not self.file_id: - raise ValueError(404, 'file_id not found.') - - return self.file_id - def parse_django_session(self, session_data): # only for django 1.11.x encoded_data = base64.b64decode(session_data) @@ -113,9 +106,7 @@ def resource_check(self): Workspaces).filter_by(id=workspace_id).first() repo_id = workspace.repo_id workspace_owner = workspace.owner - self.repo_id = repo_id - self.file_path = file_path - file_id = self.get_file_id() + file_id = get_file_id(repo_id, file_path) thumbnail_dir = os.path.join(settings.THUMBNAIL_DIR, str(size)) thumbnail_path = os.path.join(thumbnail_dir, file_id) @@ -160,13 +151,6 @@ def gen_plugin_info(self): plugin_info.update(self.resource) self.plugin_info = plugin_info - def get_file_id(self): - self.file_id = seafile_api.get_file_id_by_path(self.repo_id, self.file_path) - if not self.file_id: - raise ValueError(404, 'file_id not found.') - - return self.file_id - def params_check(self): plugin_name = self.request.url.split('/')[1] path = self.request.query_dict['path'][0] @@ -191,9 +175,7 @@ def resource_check(self): file_path ='/' + plugin.name + path file_name = os.path.basename(file_path) - self.repo_id = settings.PLUGINS_REPO_ID - self.file_path = file_path - file_id = self.get_file_id() + file_id = get_file_id(settings.PLUGINS_REPO_ID, file_path) file_info = json.loads(plugin.info) last_modified_time = datetime.strptime(file_info['last_modified'][:-6], '%Y-%m-%dT%H:%M:%S') diff --git a/seatable_thumbnail/thumbnail.py b/seatable_thumbnail/thumbnail.py index 002f067..d37584a 100644 --- a/seatable_thumbnail/thumbnail.py +++ b/seatable_thumbnail/thumbnail.py @@ -8,10 +8,11 @@ from PIL import Image from email.utils import formatdate -from seaserv import get_repo, get_file_size, seafile_api +from seaserv import get_repo, get_file_size import seatable_thumbnail.settings as settings from seatable_thumbnail.constants import IMAGE_MODES, EMPTY_BYTES,\ THUMBNAIL_EXTENSION, IMAGE, PSD, VIDEO, XMIND +from seatable_thumbnail.utils import get_inner_path class Thumbnail(object): @@ -36,7 +37,7 @@ def generate(self): if file_size > settings.THUMBNAIL_IMAGE_SIZE_LIMIT * 1024 * 1024: raise AssertionError(400, 'file_size invalid.') - inner_path = self.get_inner_path() + inner_path = get_inner_path(self.repo_id, self.file_id, self.file_name) image_file = urllib.request.urlopen(inner_path) f = BytesIO(image_file.read()) @@ -48,7 +49,7 @@ def generate(self): from psd_tools import PSDImage tmp_psd = os.path.join(tempfile.gettempdir(), self.file_id) - inner_path = self.get_inner_path() + inner_path = get_inner_path(self.repo_id, self.file_id, self.file_name) urllib.request.urlretrieve(inner_path, tmp_psd) psd = PSDImage.open(tmp_psd) @@ -63,7 +64,7 @@ def generate(self): tmp_image_path = os.path.join( tempfile.gettempdir(), self.file_id + '.png') tmp_video = os.path.join(tempfile.gettempdir(), self.file_id) - inner_path = self.get_inner_path() + inner_path = get_inner_path(self.repo_id, self.file_id, self.file_name) urllib.request.urlretrieve(inner_path, tmp_video) clip = VideoFileClip(tmp_video) @@ -77,7 +78,7 @@ def generate(self): # ===== xmind ===== elif self.file_type == XMIND: - inner_path = self.get_inner_path() + inner_path = get_inner_path(self.repo_id, self.file_id, self.file_name) xmind_file = urllib.request.urlopen(inner_path) f = BytesIO(xmind_file.read()) xmind_zip_file = zipfile.ZipFile(f, 'r') @@ -87,16 +88,6 @@ def generate(self): image = Image.open(f) self.create_image_thumbnail(image) - def get_inner_path(self): - token = seafile_api.get_fileserver_access_token( - self.repo_id, self.file_id, 'view', '', use_onetime=True) - if not token: - raise ValueError(404, 'token not found.') - self.inner_path = '%s/files/%s/%s' % ( - settings.INNER_FILE_SERVER_ROOT.rstrip('/'), token, urllib.parse.quote(self.file_name)) - - return self.inner_path - def create_image_thumbnail(self, image): width, height = image.size image_memory_cost = width * height * 4 / 1024 / 1024 diff --git a/seatable_thumbnail/utils.py b/seatable_thumbnail/utils.py new file mode 100644 index 0000000..799ba1e --- /dev/null +++ b/seatable_thumbnail/utils.py @@ -0,0 +1,40 @@ +import urllib.request +import urllib.parse + +from seaserv import seafile_api +import seatable_thumbnail.settings as settings + + +def get_file_id(repo_id, file_path): + file_id = seafile_api.get_file_id_by_path(repo_id, file_path) + if not file_id: + raise ValueError(404, 'file_id not found.') + + return file_id + + +def get_inner_path(repo_id, file_id, file_name): + token = seafile_api.get_fileserver_access_token( + repo_id, file_id, 'view', '', use_onetime=True) + if not token: + raise ValueError(404, 'token not found.') + inner_path = '%s/files/%s/%s' % ( + settings.INNER_FILE_SERVER_ROOT.rstrip('/'), token, urllib.parse.quote(file_name)) + + return inner_path + + +def cache_check(request, info): + etag = info.get('etag') + if_none_match_headers = request.headers.get('if-none-match') + if_none_match = if_none_match_headers[0] if if_none_match_headers else '' + + last_modified = info.get('last_modified') + if_modified_since_headers = request.headers.get('if-modified-since') + if_modified_since = if_modified_since_headers[0] if if_modified_since_headers else '' + + if (if_none_match and if_none_match == etag) \ + or (if_modified_since and if_modified_since == last_modified): + return True + else: + return False