Dirty migration tool for gridfs

tools/README

@@ -1,7 +1,9 @@
-Some tools to manage trantor:
+Some tools dirty to manage trantor:
 
 - adduser. Used to add users to trantor:
 $ adduser myNick
 Password:
 
 - update. Update the cover of all the books. It might be outdated.
+
+- togridfs (23/4/2013). Migrate all files and covers to gridfs

tools/togridfs/config.go

@@ -0,0 +1,32 @@
+package main
+
+const (
+	PORT = "8080"
+
+	DB_IP      = "127.0.0.1"
+	DB_NAME    = "trantor"
+	META_COLL  = "meta"
+	BOOKS_COLL = "books"
+	TAGS_COLL  = "tags"
+	USERS_COLL = "users"
+	STATS_COLL = "statistics"
+	FS_BOOKS   = "fs_books"
+	FS_IMGS    = "fs_imgs"
+
+	PASS_SALT           = "ImperialLibSalt"
+	MINUTES_UPDATE_TAGS = 10
+	TAGS_DISPLAY        = 50
+	SEARCH_ITEMS_PAGE   = 20
+	NEW_ITEMS_PAGE      = 50
+
+	TEMPLATE_PATH = "templates/"
+	CSS_PATH      = "css/"
+	JS_PATH       = "js/"
+	IMG_PATH      = "img/"
+
+	IMG_WIDTH_BIG   = 300
+	IMG_WIDTH_SMALL = 60
+	IMG_QUALITY     = 80
+
+	STATS_CHAN_SIZE = 100
+)

tools/togridfs/cover.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"bytes"
+	"git.gitorious.org/go-pkg/epubgo.git"
+	"github.com/nfnt/resize"
+	"image"
+	"image/jpeg"
+	"io"
+	"io/ioutil"
+	"labix.org/v2/mgo"
+	"labix.org/v2/mgo/bson"
+	"log"
+	"regexp"
+	"strings"
+)
+
+func GetCover(e *epubgo.Epub, title string) (bson.ObjectId, bson.ObjectId) {
+	imgId, smallId := searchCommonCoverNames(e, title)
+	if imgId != "" {
+		return imgId, smallId
+	}
+
+	/* search for img on the text */
+	exp, _ := regexp.Compile("<ima?g.*[(src)(href)]=[\"']([^\"']*(\\.[^\\.\"']*))[\"']")
+	it, errNext := e.Spine()
+	for errNext == nil {
+		file, err := it.Open()
+		if err != nil {
+			break
+		}
+		defer file.Close()
+
+		txt, err := ioutil.ReadAll(file)
+		if err != nil {
+			break
+		}
+		res := exp.FindSubmatch(txt)
+		if res != nil {
+			href := string(res[1])
+			urlPart := strings.Split(it.Url(), "/")
+			url := strings.Join(urlPart[:len(urlPart)-1], "/")
+			if href[:3] == "../" {
+				href = href[3:]
+				url = strings.Join(urlPart[:len(urlPart)-2], "/")
+			}
+			href = strings.Replace(href, "%20", " ", -1)
+			href = strings.Replace(href, "%27", "'", -1)
+			href = strings.Replace(href, "%28", "(", -1)
+			href = strings.Replace(href, "%29", ")", -1)
+			if url == "" {
+				url = href
+			} else {
+				url = url + "/" + href
+			}
+
+			img, err := e.OpenFile(url)
+			if err == nil {
+				defer img.Close()
+				return storeImg(img, title)
+			}
+		}
+		errNext = it.Next()
+	}
+	return "", ""
+}
+
+func searchCommonCoverNames(e *epubgo.Epub, title string) (bson.ObjectId, bson.ObjectId) {
+	for _, p := range []string{"cover.jpg", "Images/cover.jpg", "cover.jpeg", "cover1.jpg", "cover1.jpeg"} {
+		img, err := e.OpenFile(p)
+		if err == nil {
+			defer img.Close()
+			return storeImg(img, title)
+		}
+	}
+	return "", ""
+}
+
+func storeImg(img io.Reader, title string) (bson.ObjectId, bson.ObjectId) {
+	/* open the files */
+	fBig, err := createCoverFile(title)
+	if err != nil {
+		log.Println("Error creating", title, ":", err.Error())
+		return "", ""
+	}
+	defer fBig.Close()
+
+	fSmall, err := createCoverFile(title + "_small")
+	if err != nil {
+		log.Println("Error creating", title+"_small", ":", err.Error())
+		return "", ""
+	}
+	defer fSmall.Close()
+
+	/* resize img */
+	var img2 bytes.Buffer
+	img1 := io.TeeReader(img, &img2)
+	jpgOptions := jpeg.Options{IMG_QUALITY}
+	imgResized, err := resizeImg(img1, IMG_WIDTH_BIG)
+	if err != nil {
+		log.Println("Error resizing big image:", err.Error())
+		return "", ""
+	}
+	err = jpeg.Encode(fBig, imgResized, &jpgOptions)
+	if err != nil {
+		log.Println("Error encoding big image:", err.Error())
+		return "", ""
+	}
+	imgSmallResized, err := resizeImg(&img2, IMG_WIDTH_SMALL)
+	if err != nil {
+		log.Println("Error resizing small image:", err.Error())
+		return "", ""
+	}
+	err = jpeg.Encode(fSmall, imgSmallResized, &jpgOptions)
+	if err != nil {
+		log.Println("Error encoding small image:", err.Error())
+		return "", ""
+	}
+
+	idBig, _ := fBig.Id().(bson.ObjectId)
+	idSmall, _ := fSmall.Id().(bson.ObjectId)
+	return idBig, idSmall
+}
+
+func createCoverFile(title string) (*mgo.GridFile, error) {
+	fs := db.GetFS(FS_IMGS)
+	return fs.Create(title + ".jpg")
+}
+
+func resizeImg(imgReader io.Reader, width uint) (image.Image, error) {
+	img, _, err := image.Decode(imgReader)
+	if err != nil {
+		return nil, err
+	}
+
+	return resize.Resize(width, 0, img, resize.NearestNeighbor), nil
+}

tools/togridfs/database.go

@@ -0,0 +1,252 @@
+package main
+
+import (
+	"crypto/md5"
+	"labix.org/v2/mgo"
+	"labix.org/v2/mgo/bson"
+	"time"
+)
+
+const (
+	META_TYPE_TAGS = "tags updated"
+)
+
+var db *DB
+
+type Book struct {
+	Id          string `bson:"_id"`
+	Title       string
+	Author      []string
+	Contributor string
+	Publisher   string
+	Description string
+	Subject     []string
+	Date        string
+	Lang        []string
+	Isbn        string
+	Type        string
+	Format      string
+	Source      string
+	Relation    string
+	Coverage    string
+	Rights      string
+	Meta        string
+	File        bson.ObjectId
+	Cover       bson.ObjectId
+	CoverSmall  bson.ObjectId
+	Active      bool
+	Keywords    []string
+	Path        string
+}
+
+type DB struct {
+	session *mgo.Session
+	meta    *mgo.Collection
+	books   *mgo.Collection
+	tags    *mgo.Collection
+	user    *mgo.Collection
+	stats   *mgo.Collection
+}
+
+func initDB() *DB {
+	var err error
+	d := new(DB)
+	d.session, err = mgo.Dial(DB_IP)
+	if err != nil {
+		panic(err)
+	}
+
+	database := d.session.DB(DB_NAME)
+	d.meta = database.C(META_COLL)
+	d.books = database.C(BOOKS_COLL)
+	d.tags = database.C(TAGS_COLL)
+	d.user = database.C(USERS_COLL)
+	d.stats = database.C(STATS_COLL)
+	return d
+}
+
+func (d *DB) Close() {
+	d.session.Close()
+}
+
+func md5Pass(pass string) []byte {
+	h := md5.New()
+	hash := h.Sum(([]byte)(PASS_SALT + pass))
+	return hash
+}
+
+func (d *DB) SetPassword(user string, pass string) error {
+	hash := md5Pass(pass)
+	return d.user.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}})
+}
+
+func (d *DB) UserValid(user string, pass string) bool {
+	hash := md5Pass(pass)
+	n, err := d.user.Find(bson.M{"user": user, "pass": hash}).Count()
+	if err != nil {
+		return false
+	}
+	return n != 0
+}
+
+func (d *DB) InsertStats(stats interface{}) error {
+	return d.stats.Insert(stats)
+}
+
+func (d *DB) InsertBook(book interface{}) error {
+	return d.books.Insert(book)
+}
+
+func (d *DB) RemoveBook(id bson.ObjectId) error {
+	return d.books.Remove(bson.M{"_id": id})
+}
+
+func (d *DB) UpdateBook(id bson.ObjectId, data interface{}, unset ...interface{}) error {
+	if len(unset) > 0 {
+		return d.books.Update(bson.M{"_id": id}, bson.M{"$set": data, "$unset": unset[0]})
+	}
+	return d.books.Update(bson.M{"_id": id}, bson.M{"$set": data})
+}
+
+func (d *DB) IncVisit(id bson.ObjectId) error {
+	return d.books.Update(bson.M{"_id": id}, bson.M{"$inc": bson.M{"VisitsCount": 1}})
+}
+
+func (d *DB) IncDownload(id bson.ObjectId) error {
+	return d.books.Update(bson.M{"_id": id}, bson.M{"$inc": bson.M{"DownloadCount": 1}})
+}
+
+/* optional parameters: length and start index
+ *
+ * Returns: list of books, number found and err
+ */
+func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) {
+	var start, length int
+	if len(r) > 0 {
+		length = r[0]
+		if len(r) > 1 {
+			start = r[1]
+		}
+	}
+	q := d.books.Find(query).Sort("-_id")
+	num, err = q.Count()
+	if err != nil {
+		return
+	}
+	if start != 0 {
+		q = q.Skip(start)
+	}
+	if length != 0 {
+		q = q.Limit(length)
+	}
+
+	err = q.All(&books)
+	for i, b := range books {
+		books[i].Id = bson.ObjectId(b.Id).Hex()
+	}
+	return
+}
+
+/* Get the most visited books
+ */
+func (d *DB) GetVisitedBooks(num int) (books []Book, err error) {
+	var q *mgo.Query
+	q = d.books.Find(bson.M{"active": true}).Sort("-VisitsCount").Limit(num)
+	err = q.All(&books)
+	for i, b := range books {
+		books[i].Id = bson.ObjectId(b.Id).Hex()
+	}
+	return
+}
+
+/* Get the most downloaded books
+ */
+func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) {
+	var q *mgo.Query
+	q = d.books.Find(bson.M{"active": true}).Sort("-DownloadCount").Limit(num)
+	err = q.All(&books)
+	for i, b := range books {
+		books[i].Id = bson.ObjectId(b.Id).Hex()
+	}
+	return
+}
+
+/* optional parameters: length and start index
+ *
+ * Returns: list of books, number found and err
+ */
+func (d *DB) GetNewBooks(r ...int) (books []Book, num int, err error) {
+	return d.GetBooks(bson.M{"$nor": []bson.M{{"active": true}}}, r...)
+}
+
+func (d *DB) BookActive(id bson.ObjectId) bool {
+	var book Book
+	err := d.books.Find(bson.M{"_id": id}).One(&book)
+	if err != nil {
+		return false
+	}
+	return book.Active
+}
+
+func (d *DB) GetFS(prefix string) *mgo.GridFS {
+	return d.session.DB(DB_NAME).GridFS(prefix)
+}
+
+func (d *DB) areTagsOutdated() bool {
+	var result struct {
+		Id bson.ObjectId `bson:"_id"`
+	}
+	err := d.meta.Find(bson.M{"type": META_TYPE_TAGS}).One(&result)
+	if err != nil {
+		return true
+	}
+
+	lastUpdate := result.Id.Time()
+	return time.Since(lastUpdate).Minutes() > MINUTES_UPDATE_TAGS
+}
+
+func (d *DB) updateTags() error {
+	_, err := d.meta.RemoveAll(bson.M{"type": META_TYPE_TAGS})
+	if err != nil {
+		return err
+	}
+
+	var mr mgo.MapReduce
+	mr.Map = "function() { " +
+		"if (this.active) { this.subject.forEach(function(s) { emit(s, 1); }); }" +
+		"}"
+	mr.Reduce = "function(tag, vals) { " +
+		"var count = 0;" +
+		"vals.forEach(function() { count += 1; });" +
+		"return count;" +
+		"}"
+	mr.Out = bson.M{"replace": TAGS_COLL}
+	_, err = d.books.Find(bson.M{"active": true}).MapReduce(&mr, nil)
+	if err != nil {
+		return err
+	}
+
+	return d.meta.Insert(bson.M{"type": META_TYPE_TAGS})
+}
+
+func (d *DB) GetTags(numTags int) ([]string, error) {
+	if d.areTagsOutdated() {
+		err := d.updateTags()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var result []struct {
+		Tag string "_id"
+	}
+	err := d.tags.Find(nil).Sort("-value").Limit(numTags).All(&result)
+	if err != nil {
+		return nil, err
+	}
+	tags := make([]string, len(result))
+	for i, r := range result {
+		tags[i] = r.Tag
+	}
+	return tags, nil
+}

tools/togridfs/togridfs.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"fmt"
+	"git.gitorious.org/go-pkg/epubgo.git"
+	"io"
+	"labix.org/v2/mgo/bson"
+	"os"
+)
+
+func main() {
+	db = initDB()
+	defer db.Close()
+	books, _, _ := db.GetBooks(bson.M{})
+	fs := db.GetFS(FS_BOOKS)
+
+	for _, book := range books {
+		if book.Path == "" {
+			fmt.Println("don't needed -- ", book.Title)
+			continue
+		}
+		fmt.Println(book.Title)
+
+		path := "books/" + book.Path
+		file, err := os.Open(path)
+		if err != nil {
+			fmt.Println("os.Open ================", err)
+			continue
+		}
+		defer file.Close()
+
+		fw, err := fs.Create(book.Title + ".epub")
+		if err != nil {
+			fmt.Println("gridfs.Create ================", err)
+			continue
+		}
+		defer fw.Close()
+
+		_, err = io.Copy(fw, file)
+		if err != nil {
+			fmt.Println("io.Copy ================", err)
+			continue
+		}
+		id, _ := fw.Id().(bson.ObjectId)
+
+		e, err := epubgo.Open(path)
+		if err != nil {
+			fmt.Println("epubgo.Open ================", err)
+			continue
+		}
+		defer e.Close()
+
+		cover, coverSmall := GetCover(e, book.Title)
+		if cover != "" {
+			db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"cover": cover, "coversmall": coverSmall, "file": id}, bson.M{"path": 1})
+		} else {
+			fmt.Println("No cover ================", book.Title)
+			db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"file": id}, bson.M{"path": 1})
+		}
+	}
+}