[WIP] libzim7 sotoki rewrite stub

rewrote preparation step:
- mostly in python
- better documentation
- external-tools (p7zip, sort) are optionnal (but faster)
- bundled domains listing tool into main program

Author: renaud gaudin

.gitignore

@@ -1,12 +1,134 @@
-*~
-*pyc
-db/
-dumps/
-\.#*
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
 build/
-db.backup/
-_work
-work
-venv/
-sotoki.egg-info/
+develop-eggs/
 dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+.DS_Store
+
+.dockerignore

CHANGELOG.md

@@ -1,3 +1,10 @@
+### 2.0.0.dev0
+
+- rewrite using python-libzim (libzim7)
+- added --list-all option to list all available stackexchange domains
+- added --preparation-only to only prepare XML files
+- faster XML dumps creation step (x5)
+
 ### 1.3.2.dev0
 
 * removed pre-generated identicons (#141)

Dockerfile

@@ -1,38 +1,23 @@
-FROM python:3.8
+FROM python:3.8-slim
 
-# Install necessary packages
-RUN apt-get update -y && \
-    apt-get install -y --no-install-recommends advancecomp libxml2-dev libxslt1-dev libbz2-dev p7zip-full gif2apng imagemagick libjpeg-dev libpng-dev locales && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+RUN apt-get update -y \
+    && apt-get install -y --no-install-recommends unzip p7zip tzdata wget \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
-# Install jpegoptim
-RUN wget http://www.kokkonen.net/tjko/src/jpegoptim-1.4.6.tar.gz && \
-    tar xvf jpegoptim-1.4.6.tar.gz && \
-    cd jpegoptim-1.4.6 && ./configure && make all install && \
-    rm -rf jpegoptim-1.4.6*
+RUN echo "UTC" >  /etc/timezone
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
 
-# Install pngquant
-RUN wget http://pngquant.org/pngquant-2.12.5-src.tar.gz && \
-    tar xvf pngquant-2.12.5-src.tar.gz && \
-    cd pngquant-2.12.5 && ./configure && make all install && \
-    rm -rf pngquant-2.12.5*
+# TEMP: install pylibzim and scraperlib through built wheels (until release)
+RUN wget http://tmp.kiwix.org/wheels/libzim-1.0.0.dev0-cp38-cp38-manylinux1_x86_64.whl \
+    && wget http://tmp.kiwix.org/wheels/zimscraperlib-1.4.0.dev0-py3-none-any.whl \
+    && pip install  --no-cache-dir *.whl
 
-# Install gifsicle
-RUN wget https://www.lcdf.org/gifsicle/gifsicle-1.92.tar.gz && \
-    tar xvf gifsicle-1.92.tar.gz && \
-    cd gifsicle-1.92 && ./configure && make all install && \
-    rm -rf gifsicle-1.92*
-
-# Install sotoki
-RUN locale-gen "en_US.UTF-8"
 COPY requirements.txt /tmp/requirements.txt
-RUN pip3 install -r /tmp/requirements.txt
-COPY . /app
-WORKDIR /app
-RUN python3 setup.py install
-WORKDIR /
-RUN rm -rf /app
+RUN pip install --no-cache-dir -U pip && pip install --no-cache-dir -r /tmp/requirements.txt
+COPY . /app/
+RUN cd /app && python setup.py install && cd - && rm -rf /app
 
-# Boot commands
-CMD sotoki ; /bin/bash
+CMD ["sotoki", "--help"]

MANIFEST.in

@@ -1,2 +1,4 @@
+graft src
 include *.md
-recursive-include sotoki *.js *.md *.txt *.sh *.css *.svg *.eot *.ttf *.woff *.woff2 *.html VERSION
+include requirements.txt
+include LICENSE

README.md

@@ -1,58 +1,41 @@
-# Sotoki
+Sotoki
+========
 
-*Stack Overflow to Kiwix*
+`sotoki` (*stackoverflow to kiwix*) is an [OpenZIM](https://github.com/openzim) scraper to create offline versions of [Stack Exchange](https://stackexchange.com) websites such as [stack overflow](https://stackoverflow.com/).
 
-The goal of this project is to create a suite of tools to create
-[zim](https://openzim.org) files required by
-[kiwix](https://kiwix.org/) reader to make available [Stack Overflow](https://stackoverflow.com/)
-offline (without access to Internet). This use stackexchange dump from [Stack Exchange Data Dump](https://archive.org/details/stackexchange)
+It is based on Stack Exchange's Data Dumps hosted by [The Internet Archive](https://archive.org/download/stackexchange/).
 
-[![PyPI](https://img.shields.io/pypi/v/sotoki.svg)](https://pypi.python.org/pypi/sotoki)
-[![Docker Build Status](https://img.shields.io/docker/build/openzim/sotoki)](https://hub.docker.com/r/openzim/sotoki)
 [![CodeFactor](https://www.codefactor.io/repository/github/openzim/sotoki/badge)](https://www.codefactor.io/repository/github/openzim/sotoki)
+[![Docker](https://img.shields.io/docker/v/openzim/sotoki?label=docker&sort=semver)](https://hub.docker.com/r/openzim/sotoki)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
+[![PyPI version shields.io](https://img.shields.io/pypi/v/sotoki.svg)](https://pypi.org/project/sotoki/)
 
-## Getting started
+## ⚠️ Warning
 
-The use of btrfs as a file system is recommended (and required for stackoverflow)
+`sotoki` is undergoing a major rewrite to use libzim7 and its python binding in order to bypass filesystem limitations seen in version `1.x`. Use tagged version until this warning is removed as **current master is not-functionnal**.
 
-Install non python dependencies:
-```bash
-sudo apt-get install jpegoptim pngquant gifsicle advancecomp python-pip python-virtualenv python-dev libxml2-dev libxslt1-dev libbz2-dev p7zip-full python-pillow gif2apng imagemagick
-```
-
-Create a virtual environment for python:
-```bash
-virtualenv --system-site-packages -p python3 ./
-```
+## Usage
 
-Activate the virtual enviroment:
-```bash
-source ./bin/activate
-```
+`sotoki` works off a `domain` that you must provide. That is the domain-name of the stackexchange website you want to scrape. Run `sotoki --list-all` to get a list of those
 
-Install this lib:
-```bash
-pip3 install sotoki
-```
+### Docker
 
-Usage:
 ```bash
-sotoki <domain> <publisher> [--directory=<dir>] [--nozim] [--tag-depth=<tag_depth>] [--threads=<threads>] [--zimpath=<zimpath>] [--reset] [--reset-images] [--clean-previous] [--nofulltextindex] [--ignoreoldsite] [--nopic] [--no-userprofile]
+docker run -v my_dir:/output openzim/sotoki sotoki --help
 ```
 
-You can use `sotoki -h` to have more explanation about these options
+### Virtualenv
 
-## Example
+`sotoki` is a Python3 software. If you are not using the [Docker](https://docker.com) image, you are advised to use it in a virtual environment to avoid installing software dependencies on your system.
 
 ```bash
-for S in `./list_all.sh`
-do
-  sotoki $S Kiwix --threads=12 --reset --clean-previous --no-userprofile
-done
+python3 -m venv env      # Create virtualenv
+source env/bin/Activate  # Activate the virtualenv
+pip3 install sotoki # Install dependencies
+sotoki --help       # Display kolibri2zim help
 ```
 
-## License
+Call `deactivate` to quit the virtual environment.
+
+See `requirements.txt` for the list of python dependencies.
 
-[GPLv3](https://www.gnu.org/licenses/gpl-3.0) or later, see
-[LICENSE](LICENSE) for more details.

requirements.txt

@@ -1,12 +1,6 @@
-Jinja2==2.11.1
-lxml>=4.5.2,<4.6
-MarkupSafe==1.1.1
-docopt==0.6.2
-python-slugify==4.0.0
-beautifulsoup4==4.9.1
-mistune>=2.0.0a3
-Pillow==7.1.1
-kiwixstorage>=0.2,<1.0
-pif==0.8.2
-zimscraperlib>=1.3.1,<1.4
-
+kiwixstorage>=0.7,<1.0
+pif>=0.8.2,<0.9
+zimscraperlib>=1.4.0.dev0,<1.5
+xml_to_dict>=0.1.6,<0.2
+cli-formatter>=1.2.0,<1.3
+py7zr>=0.16.1,<0.17

setup.py

@@ -1,10 +1,11 @@
-import pathlib
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim: ai ts=4 sts=4 et sw=4 nu
 
+import pathlib
 from setuptools import setup, find_packages
 
 root_dir = pathlib.Path(__file__).parent
-with open(root_dir.joinpath("requirements.txt"), "r") as fh:
-    requirements = fh.read()
 
 
 def read(*names, **kwargs):
@@ -14,28 +15,35 @@ def read(*names, **kwargs):
 
 setup(
     name="sotoki",
-    version=read("sotoki", "VERSION").strip(),
-    description="Make zimfile from stackexchange dump",
-    long_description=open("README.md").read(),
-    author="dattaz",
-    author_email="[email protected]",
-    url="http://github.com/kiwix/sotoki",
-    keywords="kiwix zim stackexchange offline",
-    license="GPL",
-    packages=find_packages(exclude=["contrib", "docs", "tests*"]),
+    version=read("src", "sotoki", "VERSION").strip(),
+    description="Turn StackExchange dumps into ZIM files for offline usage",
+    long_description=read("README.md"),
+    long_description_content_type="text/markdown",
+    author="Kiwix",
+    author_email="[email protected]",
+    url="https://github.com/openzim/sotoki",
+    keywords="kiwix zim offline stackechange stackoverflow",
+    license="GPLv3+",
+    packages=find_packages("src"),
+    package_dir={"": "src"},
     install_requires=[
         line.strip()
-        for line in requirements.splitlines()
+        for line in read("requirements.txt").splitlines()
         if not line.strip().startswith("#")
     ],
     zip_safe=False,
-    platforms="Linux",
     include_package_data=True,
-    entry_points={"console_scripts": ["sotoki=sotoki.sotoki:run"]},
+    entry_points={
+        "console_scripts": [
+            "sotoki=sotoki.__main__:main",
+        ]
+    },
     classifiers=[
         "Development Status :: 4 - Beta",
         "Intended Audience :: Developers",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.9",
+        "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
     ],
+    python_requires=">=3.6",
 )

src/sotoki/VERSION

@@ -0,0 +1 @@
+2.0.0.dev0