diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4ea8957 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +Dockerfile* +.git +**/__pycache__/ +/**/config.json5 +/**/log.txt +/.idea/ +/**/*.iml +/**/record.db +/logs diff --git a/.travis.yml b/.travis.yml index de24f3d..13495a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,18 +6,22 @@ services: - docker before_install: - - docker pull mcr.microsoft.com/playwright/python:v1.23.0-focal + - docker pull mcr.microsoft.com/playwright/python:focal install: - echo "do not do anything" script: - export APP_VERSION=$(python3 -c "from version import *; print(VERSION)") - - docker build --rm -t lupohan44/games_hub:latest -t lupohan44/games_hub:v${APP_VERSION} . + - docker buildx create --use + - docker buildx build --platform=linux/amd64,linux/arm64 -t lupohan44/games_hub:latest -t lupohan44/games_hub:v${APP_VERSION} . + - docker buildx build -f Dockerfile.alpine --platform=linux/amd64,linux/arm64 -t lupohan44/games_hub:alpine -t lupohan44/games_hub:alpine-v${APP_VERSION} . after_success: - if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"; docker push lupohan44/games_hub:latest; docker push lupohan44/games_hub:v${APP_VERSION}; + docker push lupohan44/games_hub:alpine; + docker push lupohan44/games_hub:alpine-v${APP_VERSION}; fi diff --git a/Dockerfile b/Dockerfile index 47d079b..1c06de6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,26 @@ -FROM mcr.microsoft.com/playwright/python:v1.23.0-focal +FROM mcr.microsoft.com/playwright/python:focal as build-deps -# Reduce the size of the image -RUN cd home && git clone https://github.com/lupohan44/GamesHub -RUN cd /home/GamesHub && pip3 install -r requirements.txt -# Need to re-install webkit after reduce the size of the image -RUN python3 -m playwright install webkit firefox chromium +RUN apt-get update \ + && apt-get --no-install-recommends install -y python3-dev python3-pip build-essential \ + && ln -s /usr/bin/pip3 /usr/bin/pip3.8 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /tmp/python_requirements + +COPY ./requirements.txt . +COPY plugins plugins + +RUN find . -name 'requirements*.txt' -print | while IFS= read -r file; do pip3 install --user --no-cache-dir -r $file || exit 1; done \ + && python3 -m playwright install webkit firefox chromium + +FROM mcr.microsoft.com/playwright/python:focal + +ENV GAMESHUB_CONF_DIR_CWD=true + +COPY --from=build-deps /root/.local /root/.local +COPY . /src + +WORKDIR /config + +ENTRYPOINT ["python", "/src/app.py"] -ENTRYPOINT cd /home/wd && python3 /home/GamesHub/app.py diff --git a/Dockerfile.alpine b/Dockerfile.alpine new file mode 100644 index 0000000..a48408d --- /dev/null +++ b/Dockerfile.alpine @@ -0,0 +1,26 @@ +# Python 3.10 chosen specifically here due to a bug in 3.11 that breaks cchardet +FROM python:3.10-alpine as build-deps + +RUN apk --no-cache add alpine-sdk python3-dev libffi-dev + +WORKDIR /tmp/python_requirements + +COPY ./requirements-no_plugin.txt . +COPY plugins plugins + +RUN rm -rf plugins/gameshub.official.scraper.steamdb \ + && find . -name 'requirements*.txt' -print0 | while read -d $'\0' file; do pip3 install --user --no-cache-dir -r $file || exit 1; done + +FROM python:3.10-alpine + +ENV GAMESHUB_CONF_DIR_CWD=true + +COPY --from=build-deps /root/.local /root/.local +COPY . /src + +RUN rm -rf /src/plugins/gameshub.official.scraper.steamdb + +WORKDIR /config + +ENTRYPOINT ["python", "/src/app.py"] + diff --git a/README.md b/README.md index 8438ddf..0df48cf 100644 --- a/README.md +++ b/README.md @@ -74,13 +74,18 @@ Provide a framework to develop plugins to scrap games from different websites an ``` 3. Run with docker ```shell - docker pull lupohan44/games_hub:latest && docker run -v $GAMES_HUB_FOLDER_NAME:/home/wd --rm lupohan44/games_hub:latest + docker pull lupohan44/games_hub:latest && docker run -v $GAMES_HUB_FOLDER_NAME:/config --rm lupohan44/games_hub:latest ``` All changes by script inside docker will be permanently save to this folder. Each enabled official plugin should create a folder in ```{WORKING_DIR}/plugins/{PLUGIN_PACKAGE_NAME}``` to store runtime files and configs 4. Change plugins' config files from step 3 5. Rerun step 3 +#### Alpine Docker +There is a smaller docker image using Alpine Linux, but it does not support the `steamdb` plugin. If you opt to use this image, you must disable the `steamdb` plugin in the config file, then you can use the `alpine` tag. + +`lupohan44/games_hub:alpine` + ## Plugins structure example ``` /var/GamesHub (working directory) diff --git a/app.py b/app.py index 63e79d3..8094b25 100644 --- a/app.py +++ b/app.py @@ -34,11 +34,11 @@ def __init__(self): def parse_config() -> Dict: - logger.info(PARSING_CONFIG_MSG % CONFIG_PATH) - if not os.path.exists(CONFIG_PATH): + logger.info(PARSING_CONFIG_MSG % GAMESHUB_CONFIG_FILE) + if not os.path.exists(GAMESHUB_CONFIG_FILE): raise Exception(CONFIG_NOT_EXIST_ERROR_MSG) - config_json = load_json(CONFIG_PATH) + config_json = load_json(GAMESHUB_CONFIG_FILE) if "update" in config_json: if "check_update" in config_json["update"]: config.update_checking.enable = config_json["update"]["check_update"] @@ -106,19 +106,19 @@ def main(argv): check_update() if len(enabled_plugins) == 0: raise Exception(NO_PLUGINS_ENABLE_ERROR_MSG) - if not os.path.exists('plugins'): + if not os.path.exists(os.path.join(GAMESHUB_SRC_DIR, 'plugins')): raise Exception(NO_PLUGINS_FOLDER_ERROR_MSG) # try to enable plugins folders_under_plugins = [] - for dirs in os.listdir("plugins"): - if os.path.isdir(os.path.join("plugins", dirs)): + for dirs in os.listdir(os.path.join(GAMESHUB_SRC_DIR, "plugins")): + if os.path.isdir(os.path.join(GAMESHUB_SRC_DIR, "plugins", dirs)): folders_under_plugins.append(dirs) if len(folders_under_plugins) == 0: raise Exception(PLUGINS_FOLDER_EMPTY_ERROR_MSG) enabled_plugins_folders = [] for plugin_name in enabled_plugins: if plugin_name in folders_under_plugins: - enabled_plugins_folders.append(os.path.join("plugins", plugin_name)) + enabled_plugins_folders.append(os.path.join(GAMESHUB_SRC_DIR, "plugins", plugin_name)) if len(enabled_plugins_folders) == 0: raise Exception(NO_PLUGINS_ENABLE_ERROR_MSG) total_plugin_count = 0 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..069d739 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,8 @@ +version: '3' +services: + gameshub: + image: lupohan44/games_hub:latest + build: . + restart: on-failure + volumes: + - ./gameshub-conf/:/config diff --git a/games_hub/message.py b/games_hub/message.py index f623cb6..13dad43 100644 --- a/games_hub/message.py +++ b/games_hub/message.py @@ -14,12 +14,12 @@ CANNOT_LOAD_PLUGIN_WARNING_MSG = "failed to load plugins %s\n error:\n %s" # error message -CONFIG_NOT_EXIST_ERROR_MSG = "Cannot read %s!" % CONFIG_PATH -TELEGRAM_REQUIRE_TOKEN_ERROR_MSG = "Cannot get token of telegram from %s!" % CONFIG_PATH -TELEGRAM_REQUIRE_CHAT_ID_ERROR_MSG = "Cannot get chat_id of telegram from %s!" % CONFIG_PATH -ASF_REQUIRE_IPC_ERROR_MSG = "Cannot get ipc of asf from %s!" % CONFIG_PATH +CONFIG_NOT_EXIST_ERROR_MSG = "Cannot read %s!" % GAMESHUB_CONFIG_FILE +TELEGRAM_REQUIRE_TOKEN_ERROR_MSG = "Cannot get token of telegram from %s!" % GAMESHUB_CONFIG_FILE +TELEGRAM_REQUIRE_CHAT_ID_ERROR_MSG = "Cannot get chat_id of telegram from %s!" % GAMESHUB_CONFIG_FILE +ASF_REQUIRE_IPC_ERROR_MSG = "Cannot get ipc of asf from %s!" % GAMESHUB_CONFIG_FILE AT_LEAST_ENABLE_ONE_FUNCTION_ERROR_MSG = "Both telegram and asf are not enabled!" -CONFIG_FILE_NEEDS_TO_BE_UPDATED = "%s's format has been changed, please change it accordingly!" % CONFIG_PATH +CONFIG_FILE_NEEDS_TO_BE_UPDATED = "%s's format has been changed, please change it accordingly!" % GAMESHUB_CONFIG_FILE GET_PAGE_SOURCE_ERROR_MSG = "Get page source error!" NO_PLUGINS_FOLDER_ERROR_MSG = "Cannot find plugins folder!" NO_PLUGINS_ENABLE_ERROR_MSG = "No plugins enabled!" diff --git a/games_hub/static.py b/games_hub/static.py index 635f4ef..f0aa03a 100644 --- a/games_hub/static.py +++ b/games_hub/static.py @@ -1,12 +1,19 @@ import logging +import os +from pathlib import Path + '''Static Variables''' PROJECT_NAME = "GamesHub" GITHUB_URL = "https://github.com/lupohan44/GamesHub" GITHUB_VERSION_URL = "https://raw.githubusercontent.com/lupohan44/GamesHub/main/version.py" GITHUB_MIRROR_VERSION_URL = "https://ghproxy.com/" + GITHUB_VERSION_URL -CONFIG_PATH = "config.json5" +# If set to true, GamesHub will store config in the current working directory as opposed to the src directory +GAMESHUB_CONF_DIR_CWD = os.environ.get("GAMESHUB_CONF_DIR_CWD", False) == "true" +GAMESHUB_SRC_DIR = Path(__file__).parent.parent +GAMESHUB_CONFIG_DIR = os.getcwd() if GAMESHUB_CONF_DIR_CWD else GAMESHUB_SRC_DIR +GAMESHUB_CONFIG_FILE = os.path.join(GAMESHUB_CONFIG_DIR, "config.json5") # log format LOG_FORMAT = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') LOG_FORMAT_WITHOUT_LEVEL_NAME = logging.Formatter('%(asctime)s - %(name)s - %(message)s') diff --git a/plugins/gameshub.official.notification.apprise/AppriseNotifier.py b/plugins/gameshub.official.notification.apprise/AppriseNotifier.py index bc68ce7..d386a23 100644 --- a/plugins/gameshub.official.notification.apprise/AppriseNotifier.py +++ b/plugins/gameshub.official.notification.apprise/AppriseNotifier.py @@ -13,8 +13,8 @@ __name__ = "Apprise Notifier" __package__ = "gameshub.official.notification.apprise" __version__ = "1.0.2" -config_example_path = os.path.join(os.path.split(os.path.realpath(__file__))[0], "config.example.json5") -config_folder = os.path.join('plugins', __package__) +config_example_path = os.path.join(GAMESHUB_SRC_DIR, os.path.split(os.path.realpath(__file__))[0], "config.example.json5") +config_folder = os.path.join(GAMESHUB_CONFIG_DIR, 'plugins', __package__) if not os.path.exists(config_folder): os.makedirs(config_folder, exist_ok=True) config_path = os.path.join(config_folder, "config.json5") diff --git a/plugins/gameshub.official.redeem.asf/ASFRedeem.py b/plugins/gameshub.official.redeem.asf/ASFRedeem.py index f2a0e2d..af73833 100644 --- a/plugins/gameshub.official.redeem.asf/ASFRedeem.py +++ b/plugins/gameshub.official.redeem.asf/ASFRedeem.py @@ -13,8 +13,8 @@ __name__ = "ASF Redeem" __package__ = "gameshub.official.redeem.asf" __version__ = "1.0.4" -config_example_path = os.path.join(os.path.split(os.path.realpath(__file__))[0], "config.example.json5") -config_folder = os.path.join('plugins', __package__) +config_example_path = os.path.join(GAMESHUB_SRC_DIR, os.path.split(os.path.realpath(__file__))[0], "config.example.json5") +config_folder = os.path.join(GAMESHUB_CONFIG_DIR, 'plugins', __package__) if not os.path.exists(config_folder): os.makedirs(config_folder, exist_ok=True) record_path = os.path.join(config_folder, "record.db") diff --git a/plugins/gameshub.official.redeem.steamclient/SteamClientRedeem.py b/plugins/gameshub.official.redeem.steamclient/SteamClientRedeem.py index 7d9d12a..fbfc5fb 100644 --- a/plugins/gameshub.official.redeem.steamclient/SteamClientRedeem.py +++ b/plugins/gameshub.official.redeem.steamclient/SteamClientRedeem.py @@ -11,8 +11,8 @@ __name__ = "SteamClient Redeem" __package__ = "gameshub.official.redeem.steamclient" __version__ = "1.0.0" -config_example_path = os.path.join(os.path.split(os.path.realpath(__file__))[0], "config.example.json5") -config_folder = os.path.join('plugins', __package__) +config_example_path = os.path.join(GAMESHUB_SRC_DIR, os.path.split(os.path.realpath(__file__))[0], "config.example.json5") +config_folder = os.path.join(GAMESHUB_CONFIG_DIR, 'plugins', __package__) if not os.path.exists(config_folder): os.makedirs(config_folder, exist_ok=True) record_path = os.path.join(config_folder, "record.db") diff --git a/plugins/gameshub.official.scraper.epic/EpicScraper.py b/plugins/gameshub.official.scraper.epic/EpicScraper.py index 1c02ee6..a4b3e19 100644 --- a/plugins/gameshub.official.scraper.epic/EpicScraper.py +++ b/plugins/gameshub.official.scraper.epic/EpicScraper.py @@ -11,8 +11,8 @@ __name__ = "EpicGamesStore Scraper" __package__ = "gameshub.official.scraper.epic" __version__ = "1.0.2" -config_example_path = os.path.join(os.path.split(os.path.realpath(__file__))[0], "config.example.json5") -config_folder = os.path.join('plugins', __package__) +config_example_path = os.path.join(GAMESHUB_SRC_DIR, os.path.split(os.path.realpath(__file__))[0], "config.example.json5") +config_folder = os.path.join(GAMESHUB_CONFIG_DIR, 'plugins', __package__) if not os.path.exists(config_folder): os.makedirs(config_folder, exist_ok=True) record_path = os.path.join(config_folder, "record.db") diff --git a/plugins/gameshub.official.scraper.keylol/KeylolScraper.py b/plugins/gameshub.official.scraper.keylol/KeylolScraper.py index 098626d..2f6a8c0 100644 --- a/plugins/gameshub.official.scraper.keylol/KeylolScraper.py +++ b/plugins/gameshub.official.scraper.keylol/KeylolScraper.py @@ -14,8 +14,8 @@ __name__ = "Keylol Scraper" __package__ = "gameshub.official.scraper.keylol" __version__ = "1.0.3" -config_example_path = os.path.join(os.path.split(os.path.realpath(__file__))[0], "config.example.json5") -config_folder = os.path.join('plugins', __package__) +config_example_path = os.path.join(GAMESHUB_SRC_DIR, os.path.split(os.path.realpath(__file__))[0], "config.example.json5") +config_folder = os.path.join(GAMESHUB_CONFIG_DIR, 'plugins', __package__) if not os.path.exists(config_folder): os.makedirs(config_folder, exist_ok=True) record_path = os.path.join(config_folder, "record.db") diff --git a/plugins/gameshub.official.scraper.reddit/RedditScraper.py b/plugins/gameshub.official.scraper.reddit/RedditScraper.py index 9650385..d9f28c4 100644 --- a/plugins/gameshub.official.scraper.reddit/RedditScraper.py +++ b/plugins/gameshub.official.scraper.reddit/RedditScraper.py @@ -14,8 +14,8 @@ __name__ = "Reddit Scraper" __package__ = "gameshub.official.scraper.reddit" __version__ = "1.0.3" -config_example_path = os.path.join(os.path.split(os.path.realpath(__file__))[0], "config.example.json5") -config_folder = os.path.join('plugins', __package__) +config_example_path = os.path.join(GAMESHUB_SRC_DIR, os.path.split(os.path.realpath(__file__))[0], "config.example.json5") +config_folder = os.path.join(GAMESHUB_CONFIG_DIR, 'plugins', __package__) if not os.path.exists(config_folder): os.makedirs(config_folder, exist_ok=True) record_path = os.path.join(config_folder, "record.db") diff --git a/plugins/gameshub.official.scraper.steamdb/SteamDBScraper.py b/plugins/gameshub.official.scraper.steamdb/SteamDBScraper.py index 7b71817..9e4c0b0 100644 --- a/plugins/gameshub.official.scraper.steamdb/SteamDBScraper.py +++ b/plugins/gameshub.official.scraper.steamdb/SteamDBScraper.py @@ -14,8 +14,8 @@ __name__ = "SteamDB Scraper" __package__ = "gameshub.official.scraper.steamdb" __version__ = "1.0.2" -config_example_path = os.path.join(os.path.split(os.path.realpath(__file__))[0], "config.example.json5") -config_folder = os.path.join('plugins', __package__) +config_example_path = os.path.join(GAMESHUB_SRC_DIR, os.path.split(os.path.realpath(__file__))[0], "config.example.json5") +config_folder = os.path.join(GAMESHUB_CONFIG_DIR, 'plugins', __package__) if not os.path.exists(config_folder): os.makedirs(config_folder, exist_ok=True) record_path = os.path.join(config_folder, "record.db") diff --git a/requirements.txt b/requirements.txt index 0ff90e1..a112a27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ schedule steam[client] requests praw -epicstore_api \ No newline at end of file +epicstore_api