Add the tools folder

.gitignore

@@ -3,8 +3,5 @@ new/
 cover/
 trantor
 .*.swp
-upload/upload
+tools/adduser/adduser
-upload/books
+tools/update/update
-upload/cover
-upload/new
-adduser/adduser

tools/README

@@ -0,0 +1,7 @@
+Some tools to manage trantor:
+
+- adduser. Used to add users to trantor:
+$ adduser myNick
+Password:
+
+- update. Update the cover of all the books. It might be outdated.

tools/update/config.go

@@ -0,0 +1,22 @@
+package main
+
+const (
+	PORT              = "8080"
+	DB_IP             = "127.0.0.1"
+	DB_NAME           = "trantor"
+	BOOKS_COLL        = "books"
+	NEW_BOOKS_COLL    = "new"
+	USERS_COLL        = "users"
+	PASS_SALT         = "ImperialLibSalt"
+	TAGS_DISPLAY      = 50
+	SEARCH_ITEMS_PAGE = 10
+	TEMPLATE_PATH     = "templates/"
+	BOOKS_PATH        = "books/"
+	COVER_PATH        = "cover/"
+	NEW_PATH          = "new/"
+	CSS_PATH          = "css/"
+	JS_PATH           = "js/"
+	IMG_PATH          = "img/"
+	RESIZE_CMD        = "/usr/bin/convert -resize 300 -quality 60 "
+	RESIZE_THUMB_CMD  = "/usr/bin/convert -resize 60 -quality 60 "
+)

tools/update/database.go

@@ -0,0 +1,214 @@
+package main
+
+import (
+	"crypto/md5"
+	"labix.org/v2/mgo"
+	"labix.org/v2/mgo/bson"
+	"sort"
+)
+
+var db *DB
+
+type Book struct {
+	Id          string `bson:"_id"`
+	Title       string
+	Author      []string
+	Contributor string
+	Publisher   string
+	Description string
+	Subject     []string
+	Date        string
+	Lang        []string
+	Type        string
+	Format      string
+	Source      string
+	Relation    string
+	Coverage    string
+	Rights      string
+	Meta        string
+	Path        string
+	Cover       string
+	CoverSmall  string
+	Active      bool
+	Keywords    []string
+}
+
+type DB struct {
+	session *mgo.Session
+	books   *mgo.Collection
+	user    *mgo.Collection
+}
+
+func initDB() *DB {
+	var err error
+	d := new(DB)
+	d.session, err = mgo.Dial(DB_IP)
+	if err != nil {
+		panic(err)
+	}
+
+	d.books = d.session.DB(DB_NAME).C(BOOKS_COLL)
+	d.user = d.session.DB(DB_NAME).C(USERS_COLL)
+	return d
+}
+
+func (d *DB) Close() {
+	d.session.Close()
+}
+
+func md5Pass(pass string) []byte {
+	h := md5.New()
+	hash := h.Sum(([]byte)(PASS_SALT + pass))
+	return hash
+}
+
+func (d *DB) SetPassword(user string, pass string) error {
+	hash := md5Pass(pass)
+	return d.user.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}})
+}
+
+func (d *DB) UserValid(user string, pass string) bool {
+	hash := md5Pass(pass)
+	n, err := d.user.Find(bson.M{"user": user, "pass": hash}).Count()
+	if err != nil {
+		return false
+	}
+	return n != 0
+}
+
+func (d *DB) InsertBook(book interface{}) error {
+	return d.books.Insert(book)
+}
+
+func (d *DB) RemoveBook(id bson.ObjectId) error {
+	return d.books.Remove(bson.M{"_id": id})
+}
+
+func (d *DB) UpdateBook(id bson.ObjectId, data interface{}) error {
+	return d.books.Update(bson.M{"_id": id}, bson.M{"$set": data})
+}
+
+func (d *DB) IncVisit(id bson.ObjectId) error {
+	return d.books.Update(bson.M{"_id": id}, bson.M{"$inc": bson.M{"VisitsCount": 1}})
+}
+
+func (d *DB) IncDownload(path string) error {
+	return d.books.Update(bson.M{"path": path}, bson.M{"$inc": bson.M{"DownloadCount": 1}})
+}
+
+/* optional parameters: length and start index
+ * 
+ * Returns: list of books, number found and err
+ */
+func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) {
+	var start, length int
+	if len(r) > 0 {
+		length = r[0]
+		if len(r) > 1 {
+			start = r[1]
+		}
+	}
+	q := d.books.Find(query).Sort("-_id")
+	num, err = q.Count()
+	if err != nil {
+		return
+	}
+	if start != 0 {
+		q = q.Skip(start)
+	}
+	if length != 0 {
+		q = q.Limit(length)
+	}
+
+	err = q.All(&books)
+	for i, b := range books {
+		books[i].Id = bson.ObjectId(b.Id).Hex()
+	}
+	return
+}
+
+/* Get the most visited books
+ */
+func (d *DB) GetVisitedBooks(num int) (books []Book, err error) {
+	var q *mgo.Query
+	q = d.books.Find(bson.M{"active": true}).Sort("-VisitsCount").Limit(num)
+	err = q.All(&books)
+	for i, b := range books {
+		books[i].Id = bson.ObjectId(b.Id).Hex()
+	}
+	return
+}
+
+/* Get the most downloaded books
+ */
+func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) {
+	var q *mgo.Query
+	q = d.books.Find(bson.M{"active": true}).Sort("-DownloadCount").Limit(num)
+	err = q.All(&books)
+	for i, b := range books {
+		books[i].Id = bson.ObjectId(b.Id).Hex()
+	}
+	return
+}
+
+/* Returns: list of books, number found and err
+ */
+func (d *DB) GetNewBooks() (books []Book, num int, err error) {
+	var q *mgo.Query
+	q = d.books.Find(bson.M{"$nor": []bson.M{{"active": true}}}).Sort("-_id")
+	num, err = q.Count()
+	if err != nil {
+		return
+	}
+
+	err = q.All(&books)
+	for i, b := range books {
+		books[i].Id = bson.ObjectId(b.Id).Hex()
+	}
+	return
+}
+
+func (d *DB) BookActive(id bson.ObjectId) bool {
+	var book Book
+	err := d.books.Find(bson.M{"_id": id}).One(&book)
+	if err != nil {
+		return false
+	}
+	return book.Active
+}
+
+type tagsList []struct {
+	Subject string "_id"
+	Count   int    "value"
+}
+
+func (t tagsList) Len() int {
+	return len(t)
+}
+func (t tagsList) Less(i, j int) bool {
+	return t[i].Count > t[j].Count
+}
+func (t tagsList) Swap(i, j int) {
+	aux := t[i]
+	t[i] = t[j]
+	t[j] = aux
+}
+
+func (d *DB) GetTags() (tagsList, error) {
+	// TODO: cache the tags
+	var mr mgo.MapReduce
+	mr.Map = "function() { " +
+		"if (this.active) { this.subject.forEach(function(s) { emit(s, 1); }); }" +
+		"}"
+	mr.Reduce = "function(tag, vals) { " +
+		"var count = 0;" +
+		"vals.forEach(function() { count += 1; });" +
+		"return count;" +
+		"}"
+	var result tagsList
+	_, err := d.books.Find(nil).MapReduce(&mr, &result)
+	if err == nil {
+		sort.Sort(result)
+	}
+	return result, err
+}

tools/update/store.go

@@ -0,0 +1,265 @@
+package main
+
+import (
+	"git.gitorious.org/go-pkg/epub.git"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+)
+
+func ParseFile(path string) (string, error) {
+	book := map[string]interface{}{}
+
+	e, err := epub.Open(NEW_PATH+path, 0)
+	if err != nil {
+		return "", err
+	}
+	defer e.Close()
+
+	title := cleanStr(strings.Join(e.Metadata(epub.EPUB_TITLE), ", "))
+	book["title"] = title
+	book["author"] = parseAuthr(e.Metadata(epub.EPUB_CREATOR))
+	book["contributor"] = cleanStr(strings.Join(e.Metadata(epub.EPUB_CONTRIB), ", "))
+	book["publisher"] = cleanStr(strings.Join(e.Metadata(epub.EPUB_PUBLISHER), ", "))
+	book["description"] = parseDescription(e.Metadata(epub.EPUB_DESCRIPTION))
+	book["subject"] = parseSubject(e.Metadata(epub.EPUB_SUBJECT))
+	book["date"] = parseDate(e.Metadata(epub.EPUB_DATE))
+	book["lang"] = e.Metadata(epub.EPUB_LANG)
+	book["type"] = strings.Join(e.Metadata(epub.EPUB_TYPE), ", ")
+	book["format"] = strings.Join(e.Metadata(epub.EPUB_FORMAT), ", ")
+	book["source"] = strings.Join(e.Metadata(epub.EPUB_SOURCE), ", ")
+	book["relation"] = strings.Join(e.Metadata(epub.EPUB_RELATION), ", ")
+	book["coverage"] = strings.Join(e.Metadata(epub.EPUB_COVERAGE), ", ")
+	book["rights"] = strings.Join(e.Metadata(epub.EPUB_RIGHTS), ", ")
+	book["meta"] = strings.Join(e.Metadata(epub.EPUB_META), ", ")
+	book["path"] = path
+	cover, coverSmall := getCover(e, title)
+	book["cover"] = cover
+	book["coversmall"] = coverSmall
+	book["keywords"] = keywords(book)
+
+	db.InsertBook(book)
+	return title, nil
+}
+
+func StoreNewFile(name string, file io.Reader) (string, error) {
+	path := storePath(name)
+	fw, err := os.Create(NEW_PATH + path)
+	if err != nil {
+		return "", err
+	}
+	defer fw.Close()
+
+	const size = 1024
+	var n int = size
+	buff := make([]byte, size)
+	for n == size {
+		n, err = file.Read(buff)
+		fw.Write(buff)
+	}
+	return path, nil
+}
+
+func StoreBook(book Book) (path string, err error) {
+	title := book.Title
+	path = validFileName(BOOKS_PATH, title, ".epub")
+
+	oldPath := NEW_PATH + book.Path
+	r, _ := utf8.DecodeRuneInString(title)
+	folder := string(r)
+	if _, err = os.Stat(BOOKS_PATH + folder); err != nil {
+		err = os.Mkdir(BOOKS_PATH+folder, os.ModePerm)
+		if err != nil {
+			log.Println("Error creating", BOOKS_PATH+folder, ":", err.Error())
+			return
+		}
+	}
+	cmd := exec.Command("mv", oldPath, BOOKS_PATH+path)
+	err = cmd.Run()
+	return
+}
+
+func DeleteBook(book Book) {
+	if book.Cover != "" {
+		os.RemoveAll(book.Cover[1:])
+	}
+	if book.CoverSmall != "" {
+		os.RemoveAll(book.CoverSmall[1:])
+	}
+	os.RemoveAll(book.Path)
+}
+
+func validFileName(path string, title string, extension string) string {
+	title = strings.Replace(title, "/", "_", -1)
+	title = strings.Replace(title, "?", "_", -1)
+	title = strings.Replace(title, "#", "_", -1)
+	r, _ := utf8.DecodeRuneInString(title)
+	folder := string(r)
+	file := folder + "/" + title + extension
+	_, err := os.Stat(path + file)
+	for i := 0; err == nil; i++ {
+		file = folder + "/" + title + "_" + strconv.Itoa(i) + extension
+		_, err = os.Stat(path + file)
+	}
+	return file
+}
+
+func storePath(name string) string {
+	path := name
+	_, err := os.Stat(NEW_PATH + path)
+	for i := 0; err == nil; i++ {
+		path = strconv.Itoa(i) + "_" + name
+		_, err = os.Stat(NEW_PATH + path)
+	}
+	return path
+}
+
+func cleanStr(str string) string {
+	str = strings.Replace(str, "'", "'", -1)
+	exp, _ := regexp.Compile("&[^;]*;")
+	str = exp.ReplaceAllString(str, "")
+	exp, _ = regexp.Compile("[ ,]*$")
+	str = exp.ReplaceAllString(str, "")
+	return str
+}
+
+func storeImg(img []byte, title, extension string) (string, string) {
+	r, _ := utf8.DecodeRuneInString(title)
+	folder := string(r)
+	if _, err := os.Stat(COVER_PATH + folder); err != nil {
+		err = os.Mkdir(COVER_PATH+folder, os.ModePerm)
+		if err != nil {
+			log.Println("Error creating", COVER_PATH+folder, ":", err.Error())
+			return "", ""
+		}
+	}
+	imgPath := validFileName(COVER_PATH, title, extension)
+
+	/* store img on disk */
+	file, err := os.Create(COVER_PATH + imgPath)
+	if err != nil {
+		log.Println("Error creating", COVER_PATH+imgPath, ":", err.Error())
+		return "", ""
+	}
+	defer file.Close()
+	file.Write(img)
+
+	/* resize img */
+	resize := append(strings.Split(RESIZE_CMD, " "), COVER_PATH+imgPath, COVER_PATH+imgPath)
+	cmd := exec.Command(resize[0], resize[1:]...)
+	cmd.Run()
+	imgPathSmall := validFileName(COVER_PATH, title, "_small"+extension)
+	resize = append(strings.Split(RESIZE_THUMB_CMD, " "), COVER_PATH+imgPath, COVER_PATH+imgPathSmall)
+	cmd = exec.Command(resize[0], resize[1:]...)
+	cmd.Run()
+	return imgPath, imgPathSmall
+}
+
+func getCover(e *epub.Epub, title string) (string, string) {
+	/* Try first common names */
+	for _, p := range []string{"cover.jpg", "Images/cover.jpg", "cover.jpeg", "cover1.jpg", "cover1.jpeg"} {
+		img := e.Data(p)
+		if len(img) != 0 {
+			return storeImg(img, title, ".jpg")
+		}
+	}
+
+	/* search for img on the text */
+	exp, _ := regexp.Compile("<ima?g.*[(src)(href)]=[\"']([^\"']*(\\.[^\\.\"']*))[\"']")
+	it := e.Iterator(epub.EITERATOR_SPINE)
+	defer it.Close()
+	var err error = nil
+	txt := it.Curr()
+	for err == nil {
+		res := exp.FindStringSubmatch(txt)
+		if res != nil {
+			urlPart := strings.Split(it.CurrUrl(), "/")
+			url := strings.Join(urlPart[:len(urlPart)-1], "/")
+			if res[1][:3] == "../" {
+				res[1] = res[1][3:]
+				url = strings.Join(urlPart[:len(urlPart)-2], "/")
+			}
+			res[1] = strings.Replace(res[1], "%20", " ", -1)
+			res[1] = strings.Replace(res[1], "%27", "'", -1)
+			res[1] = strings.Replace(res[1], "%28", "(", -1)
+			res[1] = strings.Replace(res[1], "%29", ")", -1)
+			if url == "" {
+				url = res[1]
+			} else {
+				url = url + "/" + res[1]
+			}
+
+			img := e.Data(url)
+			if len(img) != 0 {
+				return storeImg(img, title, res[2])
+			}
+		}
+		txt, err = it.Next()
+	}
+	return "", ""
+}
+
+func parseAuthr(creator []string) []string {
+	exp1, _ := regexp.Compile("^(.*\\( *([^\\)]*) *\\))*$")
+	exp2, _ := regexp.Compile("^[^:]*: *(.*)$")
+	res := make([]string, len(creator))
+	for i, s := range creator {
+		auth := exp1.FindStringSubmatch(s)
+		if auth != nil {
+			res[i] = cleanStr(strings.Join(auth[2:], ", "))
+		} else {
+			auth := exp2.FindStringSubmatch(s)
+			if auth != nil {
+				res[i] = cleanStr(auth[1])
+			} else {
+				res[i] = cleanStr(s)
+			}
+		}
+	}
+	return res
+}
+
+func parseDescription(description []string) string {
+	str := cleanStr(strings.Join(description, ", "))
+	exp, _ := regexp.Compile("<[^>]*>")
+	str = exp.ReplaceAllString(str, "")
+	str = strings.Replace(str, "&amp;", "&", -1)
+	str = strings.Replace(str, "&lt;", "<", -1)
+	str = strings.Replace(str, "&gt;", ">", -1)
+	str = strings.Replace(str, "\\n", "\n", -1)
+	return str
+}
+
+func parseSubject(subject []string) []string {
+	var res []string
+	for _, s := range subject {
+		res = append(res, strings.Split(s, " / ")...)
+	}
+	return res
+}
+
+func parseDate(date []string) string {
+	if len(date) == 0 {
+		return ""
+	}
+	return strings.Replace(date[0], "Unspecified: ", "", -1)
+}
+
+func keywords(b map[string]interface{}) (k []string) {
+	title, _ := b["title"].(string)
+	k = strings.Split(title, " ")
+	author, _ := b["author"].([]string)
+	for _, a := range author {
+		k = append(k, strings.Split(a, " ")...)
+	}
+	publisher, _ := b["publisher"].(string)
+	k = append(k, strings.Split(publisher, " ")...)
+	subject, _ := b["subject"].([]string)
+	k = append(k, subject...)
+	return
+}

tools/update/update.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"fmt"
+	"git.gitorious.org/go-pkg/epub.git"
+	"labix.org/v2/mgo/bson"
+)
+
+func main() {
+	db = initDB()
+	defer db.Close()
+	books, _, _ := db.GetBooks(bson.M{})
+
+	for _, book := range books {
+		fmt.Println(book.Title)
+		e, err := epub.Open(BOOKS_PATH+book.Path, 0)
+		if err != nil {
+			fmt.Println("================", err)
+		}
+
+		cover, coverSmall := getCover(e, book.Title)
+		if cover != "" {
+			db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"cover": cover, "coversmall": coverSmall})
+		}
+		e.Close()
+	}
+}