Added a backup tool to import/export store data

Author: marcjamot

api/restapi.py

@@ -1,4 +1,5 @@
 __author__ = 'chris'
+import backupTool
 import json
 import os
 from txrestapi.resource import APIResource
@@ -601,3 +602,25 @@ def upload_image(self, request):
             request.write(json.dumps({"success": False, "reason": e.message}, indent=4))
             request.finish()
             return server.NOT_DONE_YET
+
+    @POST('^/api/v1/backup_files')
+    def backup_files(self, request):
+        output = request.args["output"][0]
+        return backupTool.backupFiles(output)
+
+    @POST('^/api/v1/export_database')
+    def export_database(self, request):
+        tables_and_columns = request.args["tables_and_columns"][0]
+        remove_previous = request.args["remove_previous"][0]
+        return backupTool.exportDatabase(tables_and_columns, remove_previous)
+
+    @POST('^/api/v1/restore_files')
+    def restore_files(self, request):
+        input = request.args["input"][0]
+        remove_previous_database_files = request.args["remove_previous_database_files"][0]
+        return backupTool.restoreFiles(input, remove_previous_database_files)
+
+    @POST('^/api/v1/import_database')
+    def import_database(self, request):
+        remove_previous = request.args["remove_previous"][0]
+        return backupTool.importDatabase(remove_previous)

backupTool.py

@@ -0,0 +1,148 @@
+__author__ = 'marc'
+from constants import DATA_FOLDER
+import csv
+import db.datastore as db
+import errno
+import os
+import re
+import shutil
+import sqlite3 as lite
+import tarfile
+import time
+
+_TABLES = {
+    'hashmap': ['hash', 'filepath'],
+    'profile': ['id', 'serializedUserInfo'],
+    'listings': ['id', 'serializedListings'],
+    'keys': ['type', 'privkey', 'pubkey'],
+    'followers': ['id', 'serializedFollowers'],
+    'following': ['id', 'serializedFollowing'],
+    'messages': ['guid', 'handle', 'signed_pubkey', 'encryption_pubkey', 'subject', 'message_type', 'message', 'timestamp', 'avatar_hash', 'signature', 'outgoing'],
+    'notifications': ['guid', 'handle', 'message', 'timestamp', 'avatar_hash'],
+    'vendors': ['guid', 'ip', 'port', 'signedPubkey'],
+    'moderators': ['guid', 'signedPubkey', 'encryptionKey', 'encryptionSignature', 'bitcoinKey', 'bitcoinSignature', 'handle'],
+    'purchases': ['id', 'title', 'timestamp', 'btc', 'address', 'status', 'thumbnail', 'seller', 'proofSig'],
+    'sales': ['id', 'title', 'timestamp', 'btc', 'address', 'status', 'thumbnail', 'seller'],
+    'dht': ['keyword', 'id', 'value', 'birthday']
+}
+
+def _getDatabase():
+    """Retrieves the OpenBazaar database file."""
+    Database = db.Database()
+    return Database.DATABASE
+
+def silentRemove(filename):
+    """Silently removes a file if it exists."""
+    try:
+        os.remove(filename)
+    except OSError as e:
+        if e.errno != errno.ENOENT: # ENOENT: no such file or directory
+            raise
+
+def _exportDatabaseToCsv(tablesAndColumns):
+    """Reads the database for all given tables and stores them as CSV files."""
+    dbFile = _getDatabase()
+    result = None
+    with lite.connect(dbFile) as dbConnection:
+        dbConnection.text_factory = str
+        cursor = dbConnection.cursor()
+        for table in tablesAndColumns:
+            table_name = table[0]
+            table_columns = ', '.join(table[1])
+            data = cursor.execute("SELECT {0} FROM {1}".format(table_columns, table_name))
+            fileName = 'table_{0}.csv'.format(table_name)
+            filePath = os.path.join('backup', fileName)
+            with open(filePath, 'wb') as f:
+                writer = csv.writer(f)
+                writer.writerow(table[1])
+                writer.writerows(data)
+    return result
+
+def backupFiles(output=None):
+    """Archives OpenBazaar files in a single tar archive."""
+    os.chdir(DATA_FOLDER)
+
+    # Archive files
+    files = os.listdir(DATA_FOLDER)
+    if not output:
+        output = 'backup_{0}.tar.gz'.format(time.strftime('%Y-%m-%d'))
+    silentRemove(output)
+    with tarfile.open(output, 'w:gz') as tar:
+        for f in files:
+            tar.add(f)
+        tar.close()
+    return 'Success'
+
+def exportDatabase(tableList, removePrevious=False):
+    """Exports given tables to the OpenBazaar folder."""
+    os.chdir(DATA_FOLDER)
+
+    # Parse table list
+    tableList = tableList.replace(' ', '').split(',')
+    tablesAndColumns = []
+    for table in tableList:
+        if table in _TABLES:
+            tablesAndColumns.append((table, _TABLES[table]))
+        else:
+            return 'ERROR, Table not found: {0}'.format(table)
+
+    # Remove existing database files and re-make them
+    if removePrevious and os.path.exists('backup'):
+        shutil.rmtree('backup')
+    if not os.path.exists('backup'):
+        os.makedirs('backup')
+    _exportDatabaseToCsv(tablesAndColumns)
+    return 'Success'
+
+def _importCsvToTable(fileName, deleteDataFirst=False):
+    """Imports given CSV file to the database."""
+    tableName = re.search(r'table_(\w+).csv', fileName).group(1)
+    dbFile = _getDatabase()
+    with lite.connect(dbFile) as dbConnection:
+        dbConnection.text_factory = str
+        cursor = dbConnection.cursor()
+        if deleteDataFirst:
+            cursor.execute('DELETE FROM {0}'.format(tableName))
+        with open(fileName, 'rb') as f:
+            reader = csv.reader(f)
+            header = True
+            for row in reader:
+                if header:
+                    header = False
+                    columns = ', '.join(['?' for _ in row])
+                    insertsql = 'INSERT INTO {0} VALUES ({1})'.format(tableName, columns)
+                    rowlen = len(row)
+                else:
+                    if len(row) == rowlen:
+                        cursor.execute(insertsql, row)
+
+
+def restoreFiles(inputFile, removePreviousDatabaseFiles=False):
+    """Restores files of given archive to OpenBazaar folder."""
+    if not inputFile:
+        return 'Input path is needed'
+    os.chdir(DATA_FOLDER)
+
+    # Remove existing database files if any
+    if removePreviousDatabaseFiles and os.path.exists('backup'):
+        shutil.rmtree('backup')
+
+    # Unarchive files
+    with tarfile.open(inputFile, 'r:gz') as tar:
+        tar.extractall()
+
+    return 'Success'
+
+if __name__ == '__main__':
+    print 'Backup tool works as a library.'
+
+def importDatabase(deletePreviousData=False):
+    """Imports table files from the OpenBazaar folder."""
+    os.chdir(DATA_FOLDER)
+
+    # Restore database files to the database
+    if os.path.exists('backup'):
+        files = ['backup/{0}'.format(f) for f in os.listdir('backup')]
+        for f in files:
+            _importCsvToTable(f, deletePreviousData)
+    return 'Success'