diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml
new file mode 100644
index 0000000..a93b49f
--- /dev/null
+++ b/.github/workflows/pytest.yaml
@@ -0,0 +1,31 @@
+---
+name: pytest
+on: push
+
+jobs:
+ pytest:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ python-version: ["3.8", "3.9", "3.10", "3.11"]
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v3
+
+ - name: Setup Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: ${{ matrix.python-version }}
+ cache: pip
+
+ - name: Setup Elasticsearch
+ run: |
+ pushd tests/elasticsearch
+ docker compose up -d --wait
+ popd
+
+ - name: Run pytest
+ run: |
+ pip install -r requirements-dev.txt
+ pytest -v
diff --git a/.gitignore b/.gitignore
index 3817a10..d697b3f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
venv/
+media
media/
config.py
diff --git a/ash.py b/ash.py
index 039d401..f66dbb0 100644
--- a/ash.py
+++ b/ash.py
@@ -2,30 +2,40 @@
A Flask-based web server that serves Twitter Archive.
'''
-import os
+from __future__ import annotations
+
import re
import pprint
import itertools
from datetime import datetime
from functools import lru_cache
-from urllib.parse import urlparse
+from urllib.parse import urlsplit
from collections.abc import Mapping
+from collections.abc import Iterator
+
import flask
import requests
from elasticsearch import Elasticsearch
-app = flask.Flask(
- __name__,
- static_url_path='/tweet/static'
-)
-app.config.from_object('config.Config')
+class DefaultConfig:
+ T_ES_HOST = 'http://localhost:9200'
+ T_ES_INDEX = 'tweets-*,toots-*'
+ T_MEDIA_FROM = 'direct'
+ T_EXTERNAL_TWEETS = False
+
+
+app = flask.Flask(__name__, static_url_path='/tweet/static')
+app.config.from_object(DefaultConfig)
+try:
+ app.config.from_object('config.Config')
+except ImportError:
+ pass
# Set up external Tweets support
if app.config.get('T_EXTERNAL_TWEETS'):
-
# https://developer.twitter.com/en/docs/basics/authentication/api-reference/token
resp = requests.post(
'https://api.twitter.com/oauth2/token',
@@ -41,7 +51,7 @@
app.config['T_TWITTER_TOKEN'] = bearer_token
-def toot_to_tweet(status):
+def toot_to_tweet(status: dict) -> dict:
'''Transform toot to be compatible with tweet-interface'''
# Status is a tweet
if status.get('user'):
@@ -72,36 +82,33 @@ def toot_to_tweet(status):
class TweetsDatabase(Mapping):
- def __init__(self, es_host, es_index):
+ def __init__(self, es_host: str, es_index: str) -> None:
self.es = Elasticsearch(es_host)
self.es_index = es_index
- def _search(self, **kwargs):
+ def _search(self, **kwargs) -> Iterator[dict]:
if not kwargs.get('index'):
kwargs['index'] = self.es_index
hits = self.es.search(**kwargs)['hits']['hits']
- tweets = []
for hit in hits:
tweet = hit['_source']
tweet['@index'] = hit['_index']
tweet = toot_to_tweet(tweet)
- tweets.append(tweet)
- return tweets
+ yield tweet
- def __getitem__(self, tweet_id):
+ def __getitem__(self, tweet_id: str | int) -> dict:
resp = self._search(
query={
'term': {
'_id': tweet_id
}
})
- if len(resp) == 0:
- raise KeyError(f'Tweet ID {tweet_id} not found')
- else:
- tweet = resp[0]
- return tweet
+ try:
+ return next(resp)
+ except StopIteration:
+ raise KeyError(f'Tweet ID {tweet_id} not found') from None
- def __iter__(self):
+ def __iter__(self) -> Iterator[int]:
resp = self._search(
sort=['@timestamp'],
#size=1000,
@@ -109,7 +116,7 @@ def __iter__(self):
for tweet in resp:
yield tweet['id']
- def __reversed__(self):
+ def __reversed__(self) -> Iterator[int]:
resp = self._search(
sort=[{
'@timestamp': {'order': 'desc'}
@@ -119,10 +126,10 @@ def __reversed__(self):
for tweet in resp:
yield tweet['id']
- def __len__(self):
+ def __len__(self) -> int:
return self.es.count(index=self.es_index)['count']
- def search(self, *, keyword=None, user_screen_name=None, index=None, limit=100):
+ def search(self, *, keyword=None, user_screen_name=None, index=None, limit=100) -> Iterator[dict]:
keyword_query = {
'simple_query_string': {
'query': keyword,
@@ -132,7 +139,7 @@ def search(self, *, keyword=None, user_screen_name=None, index=None, limit=100):
}
if user_screen_name and '@' in user_screen_name: # Mastodon
screen_name_field = 'account.fqn.keyword'
- else:
+ else: # Twitter
screen_name_field = 'user.screen_name.keyword'
user_query = {
'term': {
@@ -156,7 +163,7 @@ def search(self, *, keyword=None, user_screen_name=None, index=None, limit=100):
)
return resp
- def get_users(self):
+ def get_users(self) -> Iterator[dict]:
agg_name_twitter = 'user_screen_names'
agg_name_mastodon = 'account_fqn'
resp = self.es.search(
@@ -176,16 +183,14 @@ def get_users(self):
},
)
buckets = resp['aggregations'][agg_name_twitter]['buckets'] + resp['aggregations'][agg_name_mastodon]['buckets']
- users = [
- {
+ for bucket in buckets:
+ user = {
'screen_name': bucket['key'],
'tweets_count': bucket['doc_count']
}
- for bucket in buckets
- ]
- return users
+ yield user
- def get_indexes(self):
+ def get_indexes(self) -> Iterator[dict]:
agg_name = 'index_names'
resp = self.es.search(
index=self.es_index,
@@ -198,17 +203,15 @@ def get_indexes(self):
}
},
)
- indexes = [
- {
+ for bucket in resp['aggregations'][agg_name]['buckets']:
+ index = {
'name': bucket['key'],
'tweets_count': bucket['doc_count']
}
- for bucket in resp['aggregations'][agg_name]['buckets']
- ]
- return indexes
+ yield index
-def get_tdb():
+def get_tdb() -> TweetsDatabase:
if not hasattr(flask.g, 'tdb'):
flask.g.tdb = TweetsDatabase(
app.config['T_ES_HOST'],
@@ -218,7 +221,7 @@ def get_tdb():
@app.template_global('get_tweet_link')
-def get_tweet_link(screen_name, tweet_id, original_link=False):
+def get_tweet_link(screen_name: str, tweet_id: str | int, original_link: bool = False) -> str:
if original_link:
return f'https://twitter.com/{screen_name}/status/{tweet_id}'
else:
@@ -226,8 +229,7 @@ def get_tweet_link(screen_name, tweet_id, original_link=False):
@app.template_filter('format_tweet_text')
-def format_tweet_text(tweet):
-
+def format_tweet_text(tweet: dict) -> str:
try:
tweet_text = tweet['full_text']
except KeyError:
@@ -246,7 +248,7 @@ def format_tweet_text(tweet):
# A bare domain would be prepended a scheme but not a path,
# while a real URL would always have a path.
# https://docs.python.org/3/library/urllib.parse.html#url-parsing
- if urlparse(u['expanded_url']).path:
+ if urlsplit(u['expanded_url']).path:
a = f'{u["display_url"]}'
else:
a = u['display_url']
@@ -276,15 +278,13 @@ def format_tweet_text(tweet):
# true and has a valid "retweeted_status". Tweets that are ingested via
# Twitter Archive always has "retweeted" set to false (identical to a
# "traditional" RT.
- retweeted_status = tweet.get('retweeted_status')
- if retweeted_status:
+ if retweeted_status := tweet.get('retweeted_status'):
link = get_tweet_link('status', retweeted_status['id'])
a = f'RT'
tweet_text = tweet_text.replace('RT', a, 1)
# Format reblogged toot
- reblogged_status = tweet.get('reblog')
- if reblogged_status:
+ if reblogged_status := tweet.get('reblog'):
status_link = reblogged_status['url']
author = reblogged_status['account']['fqn']
author_link = reblogged_status['account']['url']
@@ -295,7 +295,7 @@ def format_tweet_text(tweet):
@app.template_filter('format_created_at')
-def format_created_at(timestamp, fmt):
+def format_created_at(timestamp: str, fmt: str) -> str:
try:
dt = datetime.strptime(timestamp, '%a %b %d %H:%M:%S %z %Y')
except ValueError:
@@ -307,7 +307,7 @@ def format_created_at(timestamp, fmt):
@app.template_filter('in_reply_to_link')
-def in_reply_to_link(tweet):
+def in_reply_to_link(tweet: dict) -> str:
if tweet.get('account'): # Mastodon
# If this is a self-thread, return local link
if tweet['in_reply_to_account_id'] == tweet['account']['id']:
@@ -319,12 +319,13 @@ def in_reply_to_link(tweet):
return get_tweet_link('status', tweet['in_reply_to_status_id'])
-def replace_media_url(url):
- media_key = os.path.basename(url)
+def replace_media_url(url: str) -> str:
if app.config['T_MEDIA_FROM'] == 'direct':
return url
elif app.config['T_MEDIA_FROM'] == 'filesystem':
- return flask.url_for('get_media', filename=media_key)
+ parts = urlsplit(url)
+ fs_path = f'{parts.netloc}{parts.path}'
+ return flask.url_for('get_media_from_filesystem', fs_path=fs_path)
elif app.config['T_MEDIA_FROM'] == 'mirror':
mirrors = app.config.get('T_MEDIA_MIRRORS', {})
for orig, repl in mirrors.items():
@@ -332,6 +333,8 @@ def replace_media_url(url):
return url.replace(orig, repl)
else:
return url
+ else:
+ return url
@app.route('/')
@@ -341,11 +344,9 @@ def root():
@app.route('/tweet/')
def index():
-
tdb = get_tdb()
total_tweets = len(tdb)
- default_user = app.config.get('T_DEFAULT_USER')
- if default_user:
+ if default_user := app.config.get('T_DEFAULT_USER'):
latest_tweets = tdb.search(keyword='*', user_screen_name=default_user, limit=10)
else:
latest_tweets = [tdb[tid] for tid in itertools.islice(reversed(tdb), 10)]
@@ -361,10 +362,8 @@ def index():
@lru_cache(maxsize=1024)
-def fetch_tweet(tweet_id):
-
+def fetch_tweet(tweet_id: int | str) -> dict:
token = app.config['T_TWITTER_TOKEN']
-
resp = requests.get(
'https://api.twitter.com/1.1/statuses/show.json',
headers={
@@ -384,7 +383,6 @@ def fetch_tweet(tweet_id):
@app.route('/tweet/.')
def get_tweet(tweet_id, ext):
-
if ext not in ('txt', 'json', 'html'):
flask.abort(404)
@@ -413,7 +411,7 @@ def get_tweet(tweet_id, ext):
# HTML output
- # Extract list images
+ # Extract media
images = []
videos = []
try:
@@ -423,7 +421,7 @@ def get_tweet(tweet_id, ext):
entities = tweet['entities']
media = entities.get('media', [])
for m in media:
- # type = video
+ # type is video
if m.get('type') == 'video':
variants = m['video_info']['variants']
hq_variant = max(variants, key=lambda v: v.get('bitrate', -1))
@@ -433,7 +431,7 @@ def get_tweet(tweet_id, ext):
videos.append({
'url': media_url,
})
- # type = photo
+ # type is photo
elif m.get('type') == 'photo':
media_url = m['media_url_https']
if not _is_external_tweet:
@@ -442,7 +440,7 @@ def get_tweet(tweet_id, ext):
'url': media_url,
'description': m.get('description', '')
})
- # type = unknown
+ # type is unknown
else:
pass
@@ -458,13 +456,13 @@ def get_tweet(tweet_id, ext):
return resp
-@app.route('/tweet/media/')
-def get_media(filename):
- return flask.send_from_directory(app.config['T_MEDIA_FS_PATH'], filename)
+@app.route('/tweet/media/')
+def get_media_from_filesystem(fs_path: str):
+ return flask.send_from_directory(app.config['T_MEDIA_FS_PATH'], fs_path)
@app.route('/tweet/search.')
-def search_tweet(ext):
+def search_tweet(ext: str):
if ext not in ('html', 'txt', 'json'):
flask.abort(404)
@@ -480,9 +478,8 @@ def search_tweet(ext):
indexes = tdb.get_indexes()
user = flask.request.args.get('u', '')
- keyword = flask.request.args.get('q', '')
index = flask.request.args.get('i', '')
- if keyword:
+ if keyword := flask.request.args.get('q', ''):
tweets = tdb.search(
keyword=keyword,
user_screen_name=user,
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 0000000..a635c5c
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,2 @@
+[pytest]
+pythonpath = .
diff --git a/requirements-dev.txt b/requirements-dev.txt
new file mode 100644
index 0000000..26b77f6
--- /dev/null
+++ b/requirements-dev.txt
@@ -0,0 +1,2 @@
+-r requirements.txt
+pytest
diff --git a/requirements.txt b/requirements.txt
index af1063a..b7e4f37 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
-Flask==2.1.2
-elasticsearch==8.2.0
-requests==2.27.1
+Flask==2.3.2
+elasticsearch==8.8.0
+requests==2.31.0
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..3736bc6
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,49 @@
+import os
+import json
+import time
+from pathlib import Path
+from datetime import datetime
+
+import pytest
+from elasticsearch import Elasticsearch
+
+from ash import app
+
+
+class Config:
+ TESTING = True
+ T_MEDIA_FROM = 'direct'
+
+
+@pytest.fixture
+def client(es_host, es_index):
+ app.config.from_object(Config)
+ app.config.update({
+ 'T_ES_HOST': es_host,
+ 'T_ES_INDEX': es_index,
+ })
+ return app.test_client()
+
+
+@pytest.fixture(scope='session')
+def es_host() -> str:
+ return os.environ.get('T_ES_HOST', 'http://localhost:9200')
+
+
+@pytest.fixture(scope='session')
+def es_index(es_host: str) -> str:
+ cluster = Elasticsearch(es_host)
+ now = datetime.now().strftime('%s')
+ index = f'pytest-{now}'
+ here = Path(os.path.abspath(__file__)).parent
+ tweet_files = [
+ here / 'fixtures/tweet_with_photo.json',
+ here / 'fixtures/tweet_with_video.json',
+ ]
+ for tweet_file in tweet_files:
+ tweet = json.loads(tweet_file.read_text())
+ cluster.index(index=index, id=tweet['id'], document=tweet)
+
+ time.sleep(3)
+
+ return index
diff --git a/tests/elasticsearch/docker-compose.yaml b/tests/elasticsearch/docker-compose.yaml
new file mode 100644
index 0000000..d0efacb
--- /dev/null
+++ b/tests/elasticsearch/docker-compose.yaml
@@ -0,0 +1,17 @@
+version: '3'
+services:
+ es01:
+ image: docker.elastic.co/elasticsearch/elasticsearch:8.1.3
+ container_name: es01
+ environment:
+ - discovery.type=single-node
+ - xpack.security.enabled=false
+ - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+ ports:
+ - 9200:9200
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:9200"]
+ interval: 10s
+ timeout: 1s
+ retries: 6
+ start_period: 20s
diff --git a/tests/fixtures/tweet_with_photo.json b/tests/fixtures/tweet_with_photo.json
new file mode 100644
index 0000000..0a47194
--- /dev/null
+++ b/tests/fixtures/tweet_with_photo.json
@@ -0,0 +1 @@
+{"@index": "tweets-wzyboy", "@timestamp": "2023-01-17T19:06:46+00:00", "contributors": null, "coordinates": null, "created_at": "Tue Jan 17 19:06:46 +0000 2023", "display_text_range": [0, 75], "entities": {"hashtags": [], "media": [{"display_url": "pic.twitter.com/9dauLWrDZS", "expanded_url": "https://twitter.com/wzyboy/status/1615425412921987074/photo/1", "id": 1615425410095017984, "id_str": "1615425410095017984", "indices": [76, 99], "media_url": "http://pbs.twimg.com/media/Fmsk2gHacAAJGL0.jpg", "media_url_https": "https://pbs.twimg.com/media/Fmsk2gHacAAJGL0.jpg", "sizes": {"large": {"h": 2048, "resize": "fit", "w": 922}, "medium": {"h": 1200, "resize": "fit", "w": 540}, "small": {"h": 680, "resize": "fit", "w": 306}, "thumb": {"h": 150, "resize": "crop", "w": 150}}, "type": "photo", "url": "https://t.co/9dauLWrDZS"}], "symbols": [], "urls": [], "user_mentions": []}, "extended_entities": {"media": [{"display_url": "pic.twitter.com/9dauLWrDZS", "expanded_url": "https://twitter.com/wzyboy/status/1615425412921987074/photo/1", "id": 1615425410095017984, "id_str": "1615425410095017984", "indices": [76, 99], "media_url": "http://pbs.twimg.com/media/Fmsk2gHacAAJGL0.jpg", "media_url_https": "https://pbs.twimg.com/media/Fmsk2gHacAAJGL0.jpg", "sizes": {"large": {"h": 2048, "resize": "fit", "w": 922}, "medium": {"h": 1200, "resize": "fit", "w": 540}, "small": {"h": 680, "resize": "fit", "w": 306}, "thumb": {"h": 150, "resize": "crop", "w": 150}}, "type": "photo", "url": "https://t.co/9dauLWrDZS"}]}, "favorite_count": 0, "favorited": false, "full_text": "Todoist: please connect a keyboard to your phone and press F12 to continue. https://t.co/9dauLWrDZS", "geo": null, "id": 1615425412921987074, "id_str": "1615425412921987074", "in_reply_to_screen_name": null, "in_reply_to_status_id": null, "in_reply_to_status_id_str": null, "in_reply_to_user_id": null, "in_reply_to_user_id_str": null, "is_quote_status": false, "lang": "en", "place": null, "possibly_sensitive": false, "retweet_count": 0, "retweeted": false, "source": "Twitter for Android", "truncated": false, "user": {"contributors_enabled": false, "created_at": "Fri Jun 26 05:13:44 +0000 2009", "default_profile": false, "default_profile_image": false, "description": "Das Leben ist zu kurz, um Deutsch zu lernen. 每天给 @Uucky_Lee 洗碗。欢迎在 Fediverse 里关注我。了解 Fediverse 联邦宇宙: https://t.co/WGCqN97JAz", "entities": {"description": {"urls": [{"display_url": "wzyboy.im/post/1486.html", "expanded_url": "https://wzyboy.im/post/1486.html", "indices": [101, 124], "url": "https://t.co/WGCqN97JAz"}]}, "url": {"urls": [{"display_url": "wzyboy.im", "expanded_url": "https://wzyboy.im/", "indices": [0, 23], "url": "https://t.co/btXCkHdabG"}]}}, "favourites_count": 1608, "follow_request_sent": false, "followers_count": 5072, "following": false, "friends_count": 429, "geo_enabled": true, "has_extended_profile": true, "id": 50932982, "id_str": "50932982", "is_translation_enabled": true, "is_translator": true, "lang": null, "listed_count": 115, "location": "Vancouver, BC", "name": "@wzyboy@dabr.ca", "notifications": false, "profile_background_color": "FFFFFF", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme16/bg.gif", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme16/bg.gif", "profile_background_tile": false, "profile_banner_url": "https://pbs.twimg.com/profile_banners/50932982/1573897758", "profile_image_url": "http://pbs.twimg.com/profile_images/1195639287456391168/IAxCxK39_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/1195639287456391168/IAxCxK39_normal.jpg", "profile_link_color": "8CB2BD", "profile_sidebar_border_color": "FFFFFF", "profile_sidebar_fill_color": "424C55", "profile_text_color": "8C94A2", "profile_use_background_image": false, "protected": true, "screen_name": "wzyboy", "statuses_count": 52391, "time_zone": null, "translator_type": "badged", "url": "https://t.co/btXCkHdabG", "utc_offset": null, "verified": false, "withheld_in_countries": []}}
\ No newline at end of file
diff --git a/tests/fixtures/tweet_with_video.json b/tests/fixtures/tweet_with_video.json
new file mode 100644
index 0000000..576daf3
--- /dev/null
+++ b/tests/fixtures/tweet_with_video.json
@@ -0,0 +1 @@
+{"@index": "tweets-uucky", "@timestamp": "2022-12-03T22:41:03+00:00", "contributors": null, "coordinates": null, "created_at": "Sat Dec 03 22:41:03 +0000 2022", "display_text_range": [0, 118], "entities": {"hashtags": [], "media": [{"display_url": "pic.twitter.com/CZTKmX68wh", "expanded_url": "https://twitter.com/dodo/status/1599161556507365376/video/1", "id": 1598448776582115328, "id_str": "1598448776582115328", "indices": [95, 118], "media_url": "http://pbs.twimg.com/media/Fi7VTHnUAAE1wRT.jpg", "media_url_https": "https://pbs.twimg.com/media/Fi7VTHnUAAE1wRT.jpg", "sizes": {"large": {"h": 1080, "resize": "fit", "w": 1080}, "medium": {"h": 1080, "resize": "fit", "w": 1080}, "small": {"h": 680, "resize": "fit", "w": 680}, "thumb": {"h": 150, "resize": "crop", "w": 150}}, "source_status_id": 1599161556507365376, "source_status_id_str": "1599161556507365376", "source_user_id": 1604444052, "source_user_id_str": "1604444052", "type": "photo", "url": "https://t.co/CZTKmX68wh"}], "symbols": [], "urls": [], "user_mentions": [{"id": 1604444052, "id_str": "1604444052", "indices": [3, 8], "name": "The Dodo", "screen_name": "dodo"}, {"id": 780516438113153025, "id_str": "780516438113153025", "indices": [80, 94], "name": "PlayforStrays", "screen_name": "PlayforStrays"}]}, "extended_entities": {"media": [{"additional_media_info": {"description": "", "embeddable": true, "monetizable": true, "source_user": {"contributors_enabled": false, "created_at": "Thu Jul 18 22:19:02 +0000 2013", "default_profile": false, "default_profile_image": false, "description": "For animal people.", "entities": {"description": {"urls": []}, "url": {"urls": [{"display_url": "thedodo.com", "expanded_url": "http://www.thedodo.com", "indices": [0, 23], "url": "https://t.co/vdKLF4z50p"}]}}, "favourites_count": 20883, "follow_request_sent": false, "followers_count": 2632250, "following": false, "friends_count": 4446, "geo_enabled": true, "has_extended_profile": false, "id": 1604444052, "id_str": "1604444052", "is_translation_enabled": true, "is_translator": false, "lang": null, "listed_count": 7063, "location": "New York, NY", "name": "The Dodo", "notifications": false, "profile_background_color": "A0E6F9", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_tile": true, "profile_banner_url": "https://pbs.twimg.com/profile_banners/1604444052/1619619087", "profile_image_url": "http://pbs.twimg.com/profile_images/1542905116168056832/3QZfoNql_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/1542905116168056832/3QZfoNql_normal.jpg", "profile_link_color": "FA0011", "profile_sidebar_border_color": "FFFFFF", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "profile_use_background_image": false, "protected": false, "screen_name": "dodo", "statuses_count": 79958, "time_zone": null, "translator_type": "none", "url": "https://t.co/vdKLF4z50p", "utc_offset": null, "verified": true, "withheld_in_countries": []}, "title": " Guy Finds Starving Dog On Deserted Island "}, "display_url": "pic.twitter.com/CZTKmX68wh", "expanded_url": "https://twitter.com/dodo/status/1599161556507365376/video/1", "id": 1598448776582115328, "id_str": "1598448776582115328", "indices": [95, 118], "media_url": "http://pbs.twimg.com/media/Fi7VTHnUAAE1wRT.jpg", "media_url_https": "https://pbs.twimg.com/media/Fi7VTHnUAAE1wRT.jpg", "sizes": {"large": {"h": 1080, "resize": "fit", "w": 1080}, "medium": {"h": 1080, "resize": "fit", "w": 1080}, "small": {"h": 680, "resize": "fit", "w": 680}, "thumb": {"h": 150, "resize": "crop", "w": 150}}, "source_status_id": 1599161556507365376, "source_status_id_str": "1599161556507365376", "source_user_id": 1604444052, "source_user_id_str": "1604444052", "type": "video", "url": "https://t.co/CZTKmX68wh", "video_info": {"aspect_ratio": [1, 1], "duration_millis": 212212, "variants": [{"bitrate": 1280000, "content_type": "video/mp4", "url": "https://video.twimg.com/amplify_video/1598448776582115328/vid/720x720/mn336TwuCUiapYKo.mp4?tag=16"}, {"bitrate": 832000, "content_type": "video/mp4", "url": "https://video.twimg.com/amplify_video/1598448776582115328/vid/540x540/c3ysrVHaOS3gS4pV.mp4?tag=16"}, {"bitrate": 8768000, "content_type": "video/mp4", "url": "https://video.twimg.com/amplify_video/1598448776582115328/vid/1080x1080/uj5eNeYYAafmOFL1.mp4?tag=16"}, {"bitrate": 432000, "content_type": "video/mp4", "url": "https://video.twimg.com/amplify_video/1598448776582115328/vid/320x320/2LKcsB380YiG24b8.mp4?tag=16"}, {"content_type": "application/x-mpegURL", "url": "https://video.twimg.com/amplify_video/1598448776582115328/pl/8ThFK3LUK7Z5dwno.m3u8?tag=16&container=fmp4"}]}}]}, "favorite_count": 0, "favorited": false, "full_text": "RT @dodo: This guy found a starving dog on a beach and knew what he had to do 💙 @playforstrays https://t.co/CZTKmX68wh", "geo": null, "id": 1599171888076722176, "id_str": "1599171888076722176", "in_reply_to_screen_name": null, "in_reply_to_status_id": null, "in_reply_to_status_id_str": null, "in_reply_to_user_id": null, "in_reply_to_user_id_str": null, "is_quote_status": false, "lang": "en", "place": null, "possibly_sensitive": false, "retweet_count": 131, "retweeted": false, "retweeted_status": {"contributors": null, "coordinates": null, "created_at": "Sat Dec 03 22:00:00 +0000 2022", "display_text_range": [0, 84], "entities": {"hashtags": [], "media": [{"display_url": "pic.twitter.com/CZTKmX68wh", "expanded_url": "https://twitter.com/dodo/status/1599161556507365376/video/1", "id": 1598448776582115328, "id_str": "1598448776582115328", "indices": [85, 108], "media_url": "http://pbs.twimg.com/media/Fi7VTHnUAAE1wRT.jpg", "media_url_https": "https://pbs.twimg.com/media/Fi7VTHnUAAE1wRT.jpg", "sizes": {"large": {"h": 1080, "resize": "fit", "w": 1080}, "medium": {"h": 1080, "resize": "fit", "w": 1080}, "small": {"h": 680, "resize": "fit", "w": 680}, "thumb": {"h": 150, "resize": "crop", "w": 150}}, "type": "photo", "url": "https://t.co/CZTKmX68wh"}], "symbols": [], "urls": [], "user_mentions": [{"id": 780516438113153025, "id_str": "780516438113153025", "indices": [70, 84], "name": "PlayforStrays", "screen_name": "PlayforStrays"}]}, "extended_entities": {"media": [{"additional_media_info": {"description": "", "embeddable": true, "monetizable": true, "title": " Guy Finds Starving Dog On Deserted Island "}, "display_url": "pic.twitter.com/CZTKmX68wh", "expanded_url": "https://twitter.com/dodo/status/1599161556507365376/video/1", "id": 1598448776582115328, "id_str": "1598448776582115328", "indices": [85, 108], "media_url": "http://pbs.twimg.com/media/Fi7VTHnUAAE1wRT.jpg", "media_url_https": "https://pbs.twimg.com/media/Fi7VTHnUAAE1wRT.jpg", "sizes": {"large": {"h": 1080, "resize": "fit", "w": 1080}, "medium": {"h": 1080, "resize": "fit", "w": 1080}, "small": {"h": 680, "resize": "fit", "w": 680}, "thumb": {"h": 150, "resize": "crop", "w": 150}}, "type": "video", "url": "https://t.co/CZTKmX68wh", "video_info": {"aspect_ratio": [1, 1], "duration_millis": 212212, "variants": [{"bitrate": 1280000, "content_type": "video/mp4", "url": "https://video.twimg.com/amplify_video/1598448776582115328/vid/720x720/mn336TwuCUiapYKo.mp4?tag=16"}, {"bitrate": 832000, "content_type": "video/mp4", "url": "https://video.twimg.com/amplify_video/1598448776582115328/vid/540x540/c3ysrVHaOS3gS4pV.mp4?tag=16"}, {"bitrate": 8768000, "content_type": "video/mp4", "url": "https://video.twimg.com/amplify_video/1598448776582115328/vid/1080x1080/uj5eNeYYAafmOFL1.mp4?tag=16"}, {"bitrate": 432000, "content_type": "video/mp4", "url": "https://video.twimg.com/amplify_video/1598448776582115328/vid/320x320/2LKcsB380YiG24b8.mp4?tag=16"}, {"content_type": "application/x-mpegURL", "url": "https://video.twimg.com/amplify_video/1598448776582115328/pl/8ThFK3LUK7Z5dwno.m3u8?tag=16&container=fmp4"}]}}]}, "favorite_count": 1102, "favorited": false, "full_text": "This guy found a starving dog on a beach and knew what he had to do 💙 @playforstrays https://t.co/CZTKmX68wh", "geo": null, "id": 1599161556507365376, "id_str": "1599161556507365376", "in_reply_to_screen_name": null, "in_reply_to_status_id": null, "in_reply_to_status_id_str": null, "in_reply_to_user_id": null, "in_reply_to_user_id_str": null, "is_quote_status": false, "lang": "en", "place": null, "possibly_sensitive": false, "retweet_count": 131, "retweeted": false, "source": "Twitter Media Studio", "truncated": false, "user": {"contributors_enabled": false, "created_at": "Thu Jul 18 22:19:02 +0000 2013", "default_profile": false, "default_profile_image": false, "description": "For animal people.", "entities": {"description": {"urls": []}, "url": {"urls": [{"display_url": "thedodo.com", "expanded_url": "http://www.thedodo.com", "indices": [0, 23], "url": "https://t.co/vdKLF4z50p"}]}}, "favourites_count": 20883, "follow_request_sent": false, "followers_count": 2632250, "following": false, "friends_count": 4446, "geo_enabled": true, "has_extended_profile": false, "id": 1604444052, "id_str": "1604444052", "is_translation_enabled": true, "is_translator": false, "lang": null, "listed_count": 7063, "location": "New York, NY", "name": "The Dodo", "notifications": false, "profile_background_color": "A0E6F9", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_tile": true, "profile_banner_url": "https://pbs.twimg.com/profile_banners/1604444052/1619619087", "profile_image_url": "http://pbs.twimg.com/profile_images/1542905116168056832/3QZfoNql_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/1542905116168056832/3QZfoNql_normal.jpg", "profile_link_color": "FA0011", "profile_sidebar_border_color": "FFFFFF", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "profile_use_background_image": false, "protected": false, "screen_name": "dodo", "statuses_count": 79958, "time_zone": null, "translator_type": "none", "url": "https://t.co/vdKLF4z50p", "utc_offset": null, "verified": true, "withheld_in_countries": []}}, "source": "Twitter Web App", "truncated": false, "user": {"contributors_enabled": false, "created_at": "Sat Jan 05 08:10:03 +0000 2013", "default_profile": false, "default_profile_image": false, "description": "UX。日常推为主,摸鱼up主、@HondaJOJO 鉴定过的傻、好奇、梦多、颈椎病十级、不耐冻、漆黑意志。喜欢逛超市和公园。每天给 @wzyboy 做饭。", "entities": {"description": {"urls": []}, "url": {"urls": [{"display_url": "uucky.me", "expanded_url": "https://uucky.me", "indices": [0, 23], "url": "https://t.co/j80l9t44tU"}]}}, "favourites_count": 9164, "follow_request_sent": false, "followers_count": 2336, "following": true, "friends_count": 963, "geo_enabled": true, "has_extended_profile": true, "id": 1062473329, "id_str": "1062473329", "is_translation_enabled": false, "is_translator": false, "lang": null, "listed_count": 67, "location": "海女美術大学🍁", "name": "@uucky@o3o.ca", "notifications": true, "profile_background_color": "DBE9ED", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme17/bg.gif", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme17/bg.gif", "profile_background_tile": false, "profile_banner_url": "https://pbs.twimg.com/profile_banners/1062473329/1434218278", "profile_image_url": "http://pbs.twimg.com/profile_images/1591958226446213126/lOaklion_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/1591958226446213126/lOaklion_normal.jpg", "profile_link_color": "FF0066", "profile_sidebar_border_color": "FFFFFF", "profile_sidebar_fill_color": "E6F6F9", "profile_text_color": "333333", "profile_use_background_image": true, "protected": true, "screen_name": "Uucky_Lee", "statuses_count": 42361, "time_zone": null, "translator_type": "regular", "url": "https://t.co/j80l9t44tU", "utc_offset": null, "verified": false, "withheld_in_countries": []}}
\ No newline at end of file
diff --git a/tests/test_views.py b/tests/test_views.py
new file mode 100644
index 0000000..2ace47c
--- /dev/null
+++ b/tests/test_views.py
@@ -0,0 +1,24 @@
+class TestMediaReplacement:
+ tweet_id = '1615425412921987074'
+ media_filename = 'Fmsk2gHacAAJGL0.jpg'
+ cf_domain = 'd1111111111.cloudfront.net'
+
+ def test_direct_media(self, client):
+ client.application.config['T_MEDIA_FROM'] = 'direct'
+ resp = client.get(f'/tweet/{self.tweet_id}.html')
+ assert f'https://pbs.twimg.com/media/{self.media_filename}' in resp.text
+
+ def test_mirror_media(self, client):
+ client.application.config['T_MEDIA_FROM'] = 'mirror'
+ client.application.config['T_MEDIA_MIRRORS'] = {
+ 'pbs.twimg.com': f'{self.cf_domain}/pbs.twimg.com',
+ 'video.twimg.com': f'{self.cf_domain}/video.twimg.com',
+ }
+ resp = client.get(f'/tweet/{self.tweet_id}.html')
+ assert f'https://{self.cf_domain}/pbs.twimg.com/media/{self.media_filename}' in resp.text
+
+ def test_fs_media(self, client):
+ client.application.config['T_MEDIA_FROM'] = 'filesystem'
+ client.application.config['T_MEDIA_FS_PATH'] = './media'
+ resp = client.get(f'/tweet/{self.tweet_id}.html')
+ assert f'/tweet/media/pbs.twimg.com/media/{self.media_filename}' in resp.text