Merge pull request #21 from nii236/feature/export_uploads

adds ability to export posts and uploads

Author: schollz

.gitignore

@@ -12,9 +12,11 @@
 *.out
 
 # Intermediate build artifacts
+/*.zip
 /assets
 /bindata.go
 /go.sum
+/vendor
 
 # RWTxt databases
 *db

cmd/rwtxt/main.go

@@ -21,6 +21,7 @@ var (
 func main() {
 	var (
 		err             error
+		export        = flag.Bool("export", false, "export uploads to {{TIMESTAMP}}-uploads.zip and posts to {{TIMESTAMP}}-posts.zip")
 		resizeWidth     = flag.Int("resizewidth", -1, "image width to resize on the fly")
 		resizeOnUpload  = flag.Bool("resizeonupload", false, "resize on upload")
 		resizeOnRequest = flag.Bool("resizeonrequest", false, "resize on request")
@@ -70,6 +71,18 @@ func main() {
 		panic(err)
 	}
 
+	if *export {
+		err = fs.ExportPosts()
+		if err != nil {
+			panic(err)
+		}
+		err = fs.ExportUploads()
+		if err != nil {
+			panic(err)
+		}
+		return
+	}
+
 	config := rwtxt.Config{Private: *private,
 		ResizeWidth:     *resizeWidth,
 		ResizeOnRequest: *resizeOnRequest,

pkg/db/db.go

@@ -2,11 +2,16 @@ package db
 
 import (
 	"bufio"
+	"bytes"
 	"compress/gzip"
 	"database/sql"
 	"encoding/json"
+	"fmt"
 	"html/template"
+	"io/ioutil"
 	"os"
+	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -263,6 +268,150 @@ func (fs *FileSystem) SaveBlob(id string, name string, blob []byte) (err error)
 	return
 }
 
+// ExportPosts will save posts to {{TIMESTAMP}}-posts.gz
+func (fs *FileSystem) ExportPosts() error {
+	domains, err := fs.GetDomains()
+	if err != nil {
+		return err
+	}
+
+	dir := os.TempDir()
+	postPaths := []string{}
+	for _, domain := range domains {
+		files, err := fs.GetAll(domain)
+		if err != nil {
+			return err
+		}
+		for _, file := range files {
+			fname := (fmt.Sprintf("%s-%s.md", file.Slug, file.ID))
+			r := strings.NewReader(file.Data)
+			if err != nil {
+				return err
+			}
+			var buf bytes.Buffer
+			_, err = buf.ReadFrom(r)
+			if err != nil {
+				return err
+			}
+			err = os.MkdirAll(filepath.Join(dir, domain), os.ModePerm)
+			if err != nil {
+				return err
+			}
+			fpath := filepath.Join(dir, domain, fname)
+			err = ioutil.WriteFile(fpath, buf.Bytes(), os.ModePerm)
+			if err != nil {
+				return err
+			}
+
+			postPaths = append(postPaths, fpath)
+		}
+	}
+	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
+	for _, f := range postPaths {
+		log.Debug(f)
+	}
+	utils.ZipFiles(fmt.Sprintf("%s-posts.zip", timestamp), postPaths)
+	return nil
+
+}
+
+// ExportUploads will save uploads to {{TIMESTAMP}}-uploads.gz
+func (fs *FileSystem) ExportUploads() error {
+	dir := os.TempDir()
+	files := []string{}
+
+	ids, err := fs.GetBlobIDs()
+	if err != nil {
+		return err
+	}
+
+	for _, id := range ids {
+		name, data, _, err := fs.GetBlob(id)
+		if err != nil {
+			return err
+		}
+		fname := fmt.Sprintf("%s-%s", id, name)
+
+		r, err := gzip.NewReader(bytes.NewReader(data))
+		if err != nil {
+			return err
+		}
+		var buf bytes.Buffer
+		_, err = buf.ReadFrom(r)
+		if err != nil {
+			return err
+		}
+		fpath := filepath.Join(dir, fname)
+		err = ioutil.WriteFile(fpath, buf.Bytes(), os.ModePerm)
+		if err != nil {
+			return err
+		}
+
+		files = append(files, fpath)
+	}
+
+	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
+	for _, f := range files {
+		log.Debug(f)
+	}
+	utils.ZipFiles(fmt.Sprintf("%s-uploads.zip", timestamp), files)
+	return nil
+}
+
+// GetBlobIDs will return a list of blob ids
+func (fs *FileSystem) GetBlobIDs() ([]string, error) {
+	fs.Lock()
+	defer fs.Unlock()
+	stmt, err := fs.DB.Prepare(`SELECT id FROM blobs`)
+	if err != nil {
+		return nil, err
+	}
+	defer stmt.Close()
+
+	result := []string{}
+	rows, err := stmt.Query()
+	if err != nil {
+		return nil, err
+	}
+	for rows.Next() {
+		var id string
+		err = rows.Scan(&id)
+		if err != nil {
+			return nil, err
+		}
+		result = append(result, id)
+	}
+
+	return result, nil
+}
+
+// GetDomains will return a list of domains
+func (fs *FileSystem) GetDomains() ([]string, error) {
+	fs.Lock()
+	defer fs.Unlock()
+	stmt, err := fs.DB.Prepare(`SELECT name FROM domains`)
+	if err != nil {
+		return nil, err
+	}
+	defer stmt.Close()
+
+	result := []string{}
+	rows, err := stmt.Query()
+	if err != nil {
+		return nil, err
+	}
+	for rows.Next() {
+		var domain string
+		err = rows.Scan(&domain)
+		if err != nil {
+			return nil, err
+		}
+		result = append(result, domain)
+	}
+
+	return result, nil
+}
+
 // SaveResizedImage will save a resized image
 func (fs *FileSystem) SaveResizedImage(id string, name string, blob []byte) (err error) {
 	fs.Lock()

pkg/utils/utils.go

@@ -1,11 +1,14 @@
 package utils
 
 import (
+	"archive/zip"
 	"crypto/hmac"
 	"crypto/sha512"
 	"encoding/hex"
 	"html/template"
+	"io"
 	"math/rand"
+	"os"
 	"strings"
 	"time"
 
@@ -14,6 +17,56 @@ import (
 	blackfriday "gopkg.in/russross/blackfriday.v2"
 )
 
+// ZipFiles will zip files to filename
+func ZipFiles(filename string, files []string) error {
+
+	newZipFile, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+	defer newZipFile.Close()
+
+	zipWriter := zip.NewWriter(newZipFile)
+	defer zipWriter.Close()
+
+	// Add files to zip
+	for _, file := range files {
+
+		zipfile, err := os.Open(file)
+		if err != nil {
+			return err
+		}
+		defer zipfile.Close()
+
+		// Get the file information
+		info, err := zipfile.Stat()
+		if err != nil {
+			return err
+		}
+
+		header, err := zip.FileInfoHeader(info)
+		if err != nil {
+			return err
+		}
+
+		// Using FileInfoHeader() above only uses the basename of the file. If we want
+		// to preserve the folder structure we can overwrite this with the full path.
+		header.Name = file
+
+		// Change to deflate to gain better compression
+		// see http://golang.org/pkg/archive/zip/#pkg-constants
+		header.Method = zip.Deflate
+
+		writer, err := zipWriter.CreateHeader(header)
+		if err != nil {
+			return err
+		}
+		if _, err = io.Copy(writer, zipfile); err != nil {
+			return err
+		}
+	}
+	return nil
+}
 func RenderMarkdownToHTML(markdown string) template.HTML {
 	html := string(blackfriday.Run([]byte(markdown),
 		blackfriday.WithExtensions(