Added docker-compose setup & code to support it (amitt001#31)

Dockerfile

@@ -5,7 +5,8 @@ LABEL version='1.0.0'
 LABEL description='Pygmy(pygy.co) URL shortener'
 LABEL vendor="Amit Tripathi"
 
-RUN apt update -y && apt install python3-pip -y
+RUN apt update && apt install python3-pip -y
+RUN mkdir /var/log/pygmy
 
 WORKDIR /pygmy
 ADD ./requirements.txt /pygmy/requirements.txt

README.md

@@ -5,6 +5,7 @@
 [![Build Status](https://travis-ci.org/amitt001/pygmy.svg?branch=master)](https://travis-ci.org/amitt001/pygmy)
 ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/Django.svg)
 [![PyPI license](https://img.shields.io/pypi/l/ansicolortags.svg)](https://pypi.python.org/pypi/ansicolortags/)
+![Docker Pulls](https://img.shields.io/docker/pulls/amit19/pygmy.svg)
 [![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.me/amit19)
 
 Live version of this project @ [https://pygy.co](https://pygy.co)
@@ -26,6 +27,7 @@ If you would like to supprt the project, I can be contacted on my email of sourc
         - [Use MySQL](#use-mysql)
         - [Use Postgresql](#use-postgresql)
         - [Use SQLite](#use-sqlite)
+    - [Docker](#docker-1)
     - [Using Pygmy API](#using-pygmy-api)
     - [Create User:](#create-user)
     - [Shell Usage](#shell-usage)
@@ -65,6 +67,7 @@ The architecture is very loosely coupled which allows custom integrations easily
 - DB: PostgreSQL/MySQL/SQLite
 - Others: SQLAlchmey, JWT
 - Docker
+- Docker-compose
 
 ## Installation/Setup
 
@@ -99,6 +102,10 @@ Note:
 
 ## DB Setup:
 
+By default Pygmy uses SQLite but anoy of the SQLite, MySQL, PostgreSQL can be used. Configs are present on pygmy.cfg file in `pygmy/config` directory.
+
+Use db specific instruction below. Make sure to check and modify values in pygmy.cfg file according to your DB setup.
+
 ### Use MySQL
 
 First install `pymysql`:
@@ -112,8 +119,14 @@ Check correct port:
 Change below line in `pygmy/core/pygmy.cfg`:
 
 ```
+[database]
 engine: mysql
-url: mysql+pymysql://root:[email protected]:3306/pygmy
+url: {engine}://{user}:{password}@{host}:{port}/{db_name}
+user: root
+password: root
+host: 127.0.0.1
+port: 3306
+db_name: pygmy
 ```
 
 Enter MySQL URL
@@ -124,16 +137,42 @@ Note: Better using Mysql with version > `5.6.5` to use default value of `CURRENT
 
 ### Use Postgresql
 
-`pip install psycopg2`
+Change below line in `pygmy/core/pygmy.cfg`:
 
-`postgres://[email protected]:5432/pygmy`
+```
+[database]
+engine: postgresql
+url: {engine}://{user}:{password}@{host}:{port}/{db_name}
+user: root
+password: root
+host: 127.0.0.1
+port: 5432
+db_name: pygmy
+```
 
 ### Use SQLite
 
 SQLite is natively supported in Python
 
 `sqlite:////var/lib/pygmy/pygmy.db`
 
+```
+[database]
+engine: sqlite3
+sqlite_data_dir: data
+sqlite_db_file_name: pygmy.db
+```
+
+## Docker
+
+Docker image name: amit19/pygmy
+
+Docker image can be build by: `docker build -t amit19/pygmy .`
+
+Both Dockerfile and docker-compose file are preset at the root of the project.
+
+To use docker-compose you need to pass DB credentials in docker-compose file 
+
 ## Using Pygmy API
 
 ## Create User:
@@ -263,7 +302,7 @@ See coverage report(Coverage is bad because the coverage for integration tests i
 Thanks [batarian71](https://github.com/batarian71) for providing the logo icon.
 
 ## Donations
-If this project help you reduce time to develop, you can buy me a cup of coffee :)
+If this project help you reduce time to develop, you can help me keep this project running by donating any amount you see fit :)
 
 [![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.me/amit19)
 

TODO.rst

@@ -2,6 +2,12 @@
 TODO
 ====
 
+* Reduce image size from 550+MB to > 200MB
+
+* Complete docker compose setup
+
+* Deploy pygy.co demo website with docker compose
+
 * For same netloc same url, forward stash same url
 
 * If url pattern doesn't match. Show a 404 page

docker-compose.yml

@@ -0,0 +1,52 @@
+version: '3'
+
+services:
+
+  database:
+    image: postgres:11
+    restart: always
+    ports:
+      - "5433:5432"
+    volumes:
+       - pgdata:/var/lib/postgresql/data
+    environment:
+      POSTGRES_PASSWORD: root
+      POSTGRES_USER: root
+      POSTGRES_DB: pygmy
+
+  pygmy:
+    image: pygmy:develop
+    restart: always
+    build: .
+    # ports:
+    #  - "9119:9119"
+    links:
+      - database
+    environment:
+      - DB_PASSWORD=root
+      - DB_USER=root
+      - DB_NAME=pygmy
+      - DB_HOST=database
+      - DB_PORT=5432
+    volumes:
+     - .:/pygmy
+    command: gunicorn --log-file /var/log/pygmy/error_logs.log --access-logfile /var/log/pygmy/acclogs.log --log-level DEBUG --bind 0.0.0.0:9119 --workers 2 pygmy.rest.wsgi:app
+    depends_on:
+      - database
+
+  pygmyui:
+    image: pygmy:develop
+    restart: always
+    build: .
+    ports:
+     - "8000:8000"
+    links:
+      - pygmy
+    environment:
+      - PYGMY_API_ADDRESS=pygmy
+    command: sh -c "cd pygmyui && gunicorn --log-file /var/log/pygmy/uierror_logs.log --access-logfile /var/log/pygmy/uiacclogs.log --bind 0.0.0.0:8000 --workers 2 pygmyui.wsgi && cd .."
+    depends_on:
+      - pygmy
+
+volumes:
+  pgdata:

pygmy/config/config.py

@@ -55,6 +55,7 @@ def initialize(self):
 
         if self.database['engine'] == 'sqlite3':
             root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-            sqlite_path = root_dir + '/' + self.database['data_dir'] + '/' + self.database['file_name']
+            data_dir = self.database.get('sqlite_data_dir') or 'data'
+            file_name = self.database.get('sqlite_db_file_name') or 'pygmy.db'
+            sqlite_path = root_dir + '/' + data_dir + '/' + file_name
             self.database['url'] = 'sqlite:///{}'.format(sqlite_path)
-

pygmy/config/pygmy.cfg

@@ -8,15 +8,19 @@ short_url = 127.0.0.1
 short_url_schema = http://
 
 [database]
-file_name: pygmy.db
+# Engone can be sqlite3, postgresql, mysql
 engine: sqlite3
-data_dir = data
-# url: postgres://[email protected]:5432/pygmy
-# mysql+pymysql://root:[email protected]:3306/pygmy
-user:
-password:
-host:
-port:
+# Only required if engine is sqlite3
+sqlite_data_dir: data
+sqlite_db_file_name: pygmy.db
+url: {engine}://{user}:{password}@{host}:{port}/{db_name}
+# NOTE: Modify these fields according to your DB
+user: root
+password: root
+host: 127.0.0.1
+# Mysql default port is 3306
+port: 5432
+db_name: pygmy
 
 [auth]
 refresh_secret = _))_((*REFRESH)(*_)+_

pygmy/config/pygmy_test.cfg

@@ -8,9 +8,9 @@ short_url = 127.0.0.1
 short_url_schema = http://
 
 [database]
-file_name: pygmy_test.db
+sqlite_db_file_name: pygmy_test.db
 engine: sqlite3
-data_dir = data
+sqlite_data_dir = data
 # url: postgres://[email protected]:5432/pygmy
 # sqlite:////var/lib/pygmy/pygmy.db
 # mysql+pymysql://root:[email protected]:3306/pygmy

pygmy/database/base.py

@@ -1,3 +1,5 @@
+import os
+
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.ext.declarative import declarative_base
@@ -11,7 +13,7 @@ class BaseDatabase:
 
     def __init__(self):
         self.engine = None
-        self.url = None
+        self._db_url = None
         self.store = None
 
     def _prepare(self, url):
@@ -24,11 +26,63 @@ def commit(self):
     def abort(self):
         self.store.rollback()
 
+    # TODO: Probs, should be done in config
+    @property
+    def db_url(self):
+        """Gets the url from config file pygmy.cfg and then look up for
+        following enviorment variable. If found, replcases the values
+
+        - DB_HOST
+        - DB_PORT
+        - DB_USER
+        - DB_PASS
+        - DB_DBNAME
+
+        The reason for two ways to get DB url is support for both CLI
+        based runs and ducker-compose/kubernetes based runs
+        """
+        if self._db_url is not None:
+            return self._db_url
+
+        self._db_url = config.database['url']
+        if config.database['engine'] == 'sqlite3':
+            return self._db_url
+
+        # As in case of mysql we use pymysql, modify engine here
+        if config.database['engine'] == 'mysql':
+            engine = 'mysql+pymysql'
+        else:
+            engine = config.database['engine']
+
+
+        # Get environment variables
+        host, port, user, password, db_name = (
+                os.environ.get('DB_HOST'), os.environ.get('DB_PORT'),
+                os.environ.get('DB_USER'), os.environ.get('DB_PASSWORD'),
+                os.environ.get('DB_NAME'))
+
+        if any([host, port, user, password, db_name]):
+            log.info('Replacing config value by environment variable')
+
+        url_kw_params = {
+            'engine': engine,
+            'user': user or config.database['user'],
+            'password': password or config.database['password'],
+            'host': host or config.database['host'],
+            'port': port or config.database['port'],
+            'db_name': db_name or config.database['db_name']
+        }
+        try:
+            self._db_url = self._db_url.format(**url_kw_params)
+        except KeyError as err:
+            # Raised if one of the config is not passed
+            raise KeyError('Key: {} not set in config file'.format(err))
+
+
     def initialize(self, debug=False):
-        self.url = config.database['url']
-        log.info('DB URL: {}'.format(self.url))
-        self._prepare(self.url)
-        self.engine = create_engine(self.url, echo=debug)
+        log.info('DB URL: {}'.format(self.db_url))
+        self._prepare(self.db_url)
+        self.engine = create_engine(self.db_url, echo=debug)
         session = sessionmaker(bind=self.engine)
         self.store = session()
         self.store.commit()

pygmy/rest/shorturl.py

@@ -73,9 +73,6 @@ class ShortURLApi(MethodView):
     def get(self):
         secret = request.headers.get('secret_key')
         short_url = request.args.get('url')
-        is_valid = validate_url(short_url)
-        if is_valid is False:
-            return jsonify(dict(error='Invalid URL.')), 400
         try:
             long_url = unshorten(short_url, secret_key=secret,
                                  query_by_code=True, request=request)

pygmyui/restclient/base.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import requests
+import logging
 
 from functools import wraps
 
@@ -21,6 +22,8 @@
 FORBIDDEN = 403
 RESOURCE_EXPIRED = 410
 
+logger = logging.getLogger(__name__)
+
 
 def catch_connection_error(func):
     @wraps(func)
@@ -108,6 +111,7 @@ def call(self, path, data=None, method=None,
 
         error_object = self.error_object_from_response(response)
         if error_object is not None:
+            logger.debug('Reveived error response: %s', response.text)
             raise error_object
         return response
 

pygmyui/restclient/pygmy.py

@@ -18,6 +18,14 @@ class PygmyApiClient(Client):
     """Pygmy REST API wrapper class"""
 
     def __init__(self, rest_url, username, password, hostname, request=None):
+        """
+        Arguments:
+            rest_url {[type]} -- Pygmy API url
+            username {[type]} -- api username
+            password {[type]} -- api password
+            hostname {[type]} -- the pygmy website/ui hostname. Used to create the short url
+        """
+
         self.name = username
         self.password = password
         self.rest_url = rest_url

pygmyui/utils.py

@@ -1,6 +1,9 @@
 """A common util file to use methods across different django apps"""
+import os
+import logging
 
 from restclient.pygmy import PygmyApiClient
+
 from urllib.parse import urlparse
 
 
@@ -13,7 +16,13 @@ def make_url(url_address):
 def pygmy_client_object(config, request):
     rest_user = config.PYGMY_API_USER
     rest_pass = config.PYGMY_API_PASSWORD
-    rest_url = make_url(config.PYGMY_API_ADDRESS)
+    pygmy_api_host, pygmy_api_port = urlparse(config.PYGMY_API_ADDRESS).netloc.split(':')
+
+    # Check if PYGMY_API_ADDRESS enviornment varibale is set
+    if os.environ.get('PYGMY_API_ADDRESS'):
+        pygmy_api_host = os.environ.get('PYGMY_API_ADDRESS')
+        logging.info('Using environment variable PYGMY_API_ADDRESS. API URL: %s', pygmy_api_host)
+
+    rest_url = make_url('http://{}:{}'.format(pygmy_api_host, pygmy_api_port))
     hostname = config.HOSTNAME
     return PygmyApiClient(rest_url, rest_user, rest_pass, hostname, request)
-