Merge branch 'master' into stats

.gitignore

@@ -6,3 +6,4 @@ trantor
 tools/adduser/adduser
 tools/update/update
 tools/togridfs/togridfs
+tools/getISBNnDesc/getISBNnDesc

store.go

@@ -87,7 +87,8 @@ func parseAuthr(creator []string) []string {
 }
 
 func parseDescription(description []string) string {
-	str := cleanStr(strings.Join(description, ", "))
+	str := cleanStr(strings.Join(description, "\n"))
+	str = strings.Replace(str, "</p>", "\n", -1)
 	exp, _ := regexp.Compile("<[^>]*>")
 	str = exp.ReplaceAllString(str, "")
 	str = strings.Replace(str, "&amp;", "&", -1)

templates/book.html

@@ -60,16 +60,16 @@ function delBook(){
 		</div>
 	</div>
 	</div>
+{{end}}
 
 	<div class="row">
-	{{if .Description}}
 	<div class="span8">
-		<p>{{.Description}}</p>
-	</div>
+	{{range .Description}}
+		<p>{{.}}</p>
 	{{end}}
 	</div>
 	</div>
+	</div>
 </div>
-{{end}}
 
 {{template "footer.html"}}

tools/README

@@ -7,3 +7,5 @@ Password:
 - update. Update the cover of all the books. It might be outdated.
 
 - togridfs (23/4/2013). Migrate all files and covers to gridfs
+
+- getISBNnDesc (31/5/2013). Import the ISBN and the description with changes of lines to the database

tools/getISBNnDesc/config.go

@@ -0,0 +1,32 @@
+package main
+
+const (
+	PORT = "8080"
+
+	DB_IP      = "127.0.0.1"
+	DB_NAME    = "trantor"
+	META_COLL  = "meta"
+	BOOKS_COLL = "books"
+	TAGS_COLL  = "tags"
+	USERS_COLL = "users"
+	STATS_COLL = "statistics"
+	FS_BOOKS   = "fs_books"
+	FS_IMGS    = "fs_imgs"
+
+	PASS_SALT           = "ImperialLibSalt"
+	MINUTES_UPDATE_TAGS = 10
+	TAGS_DISPLAY        = 50
+	SEARCH_ITEMS_PAGE   = 20
+	NEW_ITEMS_PAGE      = 50
+
+	TEMPLATE_PATH = "templates/"
+	CSS_PATH      = "css/"
+	JS_PATH       = "js/"
+	IMG_PATH      = "img/"
+
+	IMG_WIDTH_BIG   = 300
+	IMG_WIDTH_SMALL = 60
+	IMG_QUALITY     = 80
+
+	CHAN_SIZE = 100
+)

tools/getISBNnDesc/database.go

@@ -0,0 +1,248 @@
+package main
+
+import (
+	"crypto/md5"
+	"labix.org/v2/mgo"
+	"labix.org/v2/mgo/bson"
+	"time"
+)
+
+const (
+	META_TYPE_TAGS = "tags updated"
+)
+
+var db *DB
+
+type Book struct {
+	Id          string `bson:"_id"`
+	Title       string
+	Author      []string
+	Contributor string
+	Publisher   string
+	Description string
+	Subject     []string
+	Date        string
+	Lang        []string
+	Isbn        string
+	Type        string
+	Format      string
+	Source      string
+	Relation    string
+	Coverage    string
+	Rights      string
+	Meta        string
+	File        bson.ObjectId
+	Cover       bson.ObjectId
+	CoverSmall  bson.ObjectId
+	Active      bool
+	Keywords    []string
+}
+
+type DB struct {
+	session *mgo.Session
+	meta    *mgo.Collection
+	books   *mgo.Collection
+	tags    *mgo.Collection
+	user    *mgo.Collection
+	stats   *mgo.Collection
+}
+
+func initDB() *DB {
+	var err error
+	d := new(DB)
+	d.session, err = mgo.Dial(DB_IP)
+	if err != nil {
+		panic(err)
+	}
+
+	database := d.session.DB(DB_NAME)
+	d.meta = database.C(META_COLL)
+	d.books = database.C(BOOKS_COLL)
+	d.tags = database.C(TAGS_COLL)
+	d.user = database.C(USERS_COLL)
+	d.stats = database.C(STATS_COLL)
+	return d
+}
+
+func (d *DB) Close() {
+	d.session.Close()
+}
+
+func md5Pass(pass string) []byte {
+	h := md5.New()
+	hash := h.Sum(([]byte)(PASS_SALT + pass))
+	return hash
+}
+
+func (d *DB) SetPassword(user string, pass string) error {
+	hash := md5Pass(pass)
+	return d.user.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}})
+}
+
+func (d *DB) UserValid(user string, pass string) bool {
+	hash := md5Pass(pass)
+	n, err := d.user.Find(bson.M{"user": user, "pass": hash}).Count()
+	if err != nil {
+		return false
+	}
+	return n != 0
+}
+
+func (d *DB) InsertStats(stats interface{}) error {
+	return d.stats.Insert(stats)
+}
+
+func (d *DB) InsertBook(book interface{}) error {
+	return d.books.Insert(book)
+}
+
+func (d *DB) RemoveBook(id bson.ObjectId) error {
+	return d.books.Remove(bson.M{"_id": id})
+}
+
+func (d *DB) UpdateBook(id bson.ObjectId, data interface{}) error {
+	return d.books.Update(bson.M{"_id": id}, bson.M{"$set": data})
+}
+
+func (d *DB) IncVisit(id bson.ObjectId) error {
+	return d.books.Update(bson.M{"_id": id}, bson.M{"$inc": bson.M{"VisitsCount": 1}})
+}
+
+func (d *DB) IncDownload(id bson.ObjectId) error {
+	return d.books.Update(bson.M{"_id": id}, bson.M{"$inc": bson.M{"DownloadCount": 1}})
+}
+
+/* optional parameters: length and start index
+ *
+ * Returns: list of books, number found and err
+ */
+func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) {
+	var start, length int
+	if len(r) > 0 {
+		length = r[0]
+		if len(r) > 1 {
+			start = r[1]
+		}
+	}
+	q := d.books.Find(query).Sort("-_id")
+	num, err = q.Count()
+	if err != nil {
+		return
+	}
+	if start != 0 {
+		q = q.Skip(start)
+	}
+	if length != 0 {
+		q = q.Limit(length)
+	}
+
+	err = q.All(&books)
+	for i, b := range books {
+		books[i].Id = bson.ObjectId(b.Id).Hex()
+	}
+	return
+}
+
+/* Get the most visited books
+ */
+func (d *DB) GetVisitedBooks(num int) (books []Book, err error) {
+	var q *mgo.Query
+	q = d.books.Find(bson.M{"active": true}).Sort("-VisitsCount").Limit(num)
+	err = q.All(&books)
+	for i, b := range books {
+		books[i].Id = bson.ObjectId(b.Id).Hex()
+	}
+	return
+}
+
+/* Get the most downloaded books
+ */
+func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) {
+	var q *mgo.Query
+	q = d.books.Find(bson.M{"active": true}).Sort("-DownloadCount").Limit(num)
+	err = q.All(&books)
+	for i, b := range books {
+		books[i].Id = bson.ObjectId(b.Id).Hex()
+	}
+	return
+}
+
+/* optional parameters: length and start index
+ *
+ * Returns: list of books, number found and err
+ */
+func (d *DB) GetNewBooks(r ...int) (books []Book, num int, err error) {
+	return d.GetBooks(bson.M{"$nor": []bson.M{{"active": true}}}, r...)
+}
+
+func (d *DB) BookActive(id bson.ObjectId) bool {
+	var book Book
+	err := d.books.Find(bson.M{"_id": id}).One(&book)
+	if err != nil {
+		return false
+	}
+	return book.Active
+}
+
+func (d *DB) GetFS(prefix string) *mgo.GridFS {
+	return d.session.DB(DB_NAME).GridFS(prefix)
+}
+
+func (d *DB) areTagsOutdated() bool {
+	var result struct {
+		Id bson.ObjectId `bson:"_id"`
+	}
+	err := d.meta.Find(bson.M{"type": META_TYPE_TAGS}).One(&result)
+	if err != nil {
+		return true
+	}
+
+	lastUpdate := result.Id.Time()
+	return time.Since(lastUpdate).Minutes() > MINUTES_UPDATE_TAGS
+}
+
+func (d *DB) updateTags() error {
+	_, err := d.meta.RemoveAll(bson.M{"type": META_TYPE_TAGS})
+	if err != nil {
+		return err
+	}
+
+	var mr mgo.MapReduce
+	mr.Map = "function() { " +
+		"if (this.active) { this.subject.forEach(function(s) { emit(s, 1); }); }" +
+		"}"
+	mr.Reduce = "function(tag, vals) { " +
+		"var count = 0;" +
+		"vals.forEach(function() { count += 1; });" +
+		"return count;" +
+		"}"
+	mr.Out = bson.M{"replace": TAGS_COLL}
+	_, err = d.books.Find(bson.M{"active": true}).MapReduce(&mr, nil)
+	if err != nil {
+		return err
+	}
+
+	return d.meta.Insert(bson.M{"type": META_TYPE_TAGS})
+}
+
+func (d *DB) GetTags(numTags int) ([]string, error) {
+	if d.areTagsOutdated() {
+		err := d.updateTags()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var result []struct {
+		Tag string "_id"
+	}
+	err := d.tags.Find(nil).Sort("-value").Limit(numTags).All(&result)
+	if err != nil {
+		return nil, err
+	}
+	tags := make([]string, len(result))
+	for i, r := range result {
+		tags[i] = r.Tag
+	}
+	return tags, nil
+}

tools/getISBNnDesc/get.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"fmt"
+	"git.gitorious.org/go-pkg/epubgo.git"
+	"labix.org/v2/mgo/bson"
+)
+
+func main() {
+	db = initDB()
+	defer db.Close()
+	books, _, _ := db.GetBooks(bson.M{})
+
+	for _, book := range books {
+		fmt.Println(book.Title)
+		e, err := OpenBook(book.File)
+		if err != nil {
+			fmt.Println("================", err)
+			continue
+		}
+
+		updateISBN(e, book)
+		updateDescription(e, book)
+		e.Close()
+	}
+}
+
+func updateISBN(e *epubgo.Epub, book Book) {
+	attr, err := e.MetadataAttr("identifier")
+	if err != nil {
+		fmt.Println("isbn ================", err)
+		return
+	}
+	data, err := e.Metadata("identifier")
+	if err != nil {
+		fmt.Println("isbn ================", err)
+		return
+	}
+	var isbn string
+	for i, d := range data {
+		if attr[i]["scheme"] == "ISBN" {
+			isbn = d
+		}
+	}
+
+	if isbn != "" {
+		db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"isbn": isbn})
+	}
+}
+
+func updateDescription(e *epubgo.Epub, book Book) {
+	descList, err := e.Metadata("description")
+	if err != nil {
+		fmt.Println("desc ================", err)
+		return
+	}
+	description := parseDescription(descList)
+	if len(description) < 10 {
+		return
+	}
+
+	if len(book.Description) < 10 || book.Description[:10] == description[:10] {
+		db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"description": description})
+	}
+}

tools/getISBNnDesc/store.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"bytes"
+	"git.gitorious.org/go-pkg/epubgo.git"
+	"io"
+	"io/ioutil"
+	"labix.org/v2/mgo/bson"
+	"regexp"
+	"strings"
+)
+
+func OpenBook(id bson.ObjectId) (*epubgo.Epub, error) {
+	fs := db.GetFS(FS_BOOKS)
+	f, err := fs.OpenId(id)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	buff, err := ioutil.ReadAll(f)
+	reader := bytes.NewReader(buff)
+
+	return epubgo.Load(reader, int64(len(buff)))
+}
+
+func StoreNewFile(name string, file io.Reader) (bson.ObjectId, error) {
+	fs := db.GetFS(FS_BOOKS)
+	fw, err := fs.Create(name)
+	if err != nil {
+		return "", err
+	}
+	defer fw.Close()
+
+	_, err = io.Copy(fw, file)
+	id, _ := fw.Id().(bson.ObjectId)
+	return id, err
+}
+
+func DeleteFile(id bson.ObjectId) error {
+	fs := db.GetFS(FS_BOOKS)
+	return fs.RemoveId(id)
+}
+
+func DeleteCover(id bson.ObjectId) error {
+	fs := db.GetFS(FS_IMGS)
+	return fs.RemoveId(id)
+}
+
+func DeleteBook(book Book) {
+	if book.Cover != "" {
+		DeleteCover(book.Cover)
+	}
+	if book.CoverSmall != "" {
+		DeleteCover(book.CoverSmall)
+	}
+	DeleteFile(book.File)
+}
+
+func cleanStr(str string) string {
+	str = strings.Replace(str, "'", "'", -1)
+	exp, _ := regexp.Compile("&[^;]*;")
+	str = exp.ReplaceAllString(str, "")
+	exp, _ = regexp.Compile("[ ,]*$")
+	str = exp.ReplaceAllString(str, "")
+	return str
+}
+
+func parseAuthr(creator []string) []string {
+	exp1, _ := regexp.Compile("^(.*\\( *([^\\)]*) *\\))*$")
+	exp2, _ := regexp.Compile("^[^:]*: *(.*)$")
+	res := make([]string, len(creator))
+	for i, s := range creator {
+		auth := exp1.FindStringSubmatch(s)
+		if auth != nil {
+			res[i] = cleanStr(strings.Join(auth[2:], ", "))
+		} else {
+			auth := exp2.FindStringSubmatch(s)
+			if auth != nil {
+				res[i] = cleanStr(auth[1])
+			} else {
+				res[i] = cleanStr(s)
+			}
+		}
+	}
+	return res
+}
+
+func parseDescription(description []string) string {
+	str := cleanStr(strings.Join(description, "\n"))
+	str = strings.Replace(str, "</p>", "\n", -1)
+	exp, _ := regexp.Compile("<[^>]*>")
+	str = exp.ReplaceAllString(str, "")
+	str = strings.Replace(str, "&amp;", "&", -1)
+	str = strings.Replace(str, "&lt;", "<", -1)
+	str = strings.Replace(str, "&gt;", ">", -1)
+	str = strings.Replace(str, "\\n", "\n", -1)
+	return str
+}
+
+func parseSubject(subject []string) []string {
+	var res []string
+	for _, s := range subject {
+		res = append(res, strings.Split(s, " / ")...)
+	}
+	return res
+}
+
+func parseDate(date []string) string {
+	if len(date) == 0 {
+		return ""
+	}
+	return strings.Replace(date[0], "Unspecified: ", "", -1)
+}
+
+func keywords(b map[string]interface{}) (k []string) {
+	title, _ := b["title"].(string)
+	k = strings.Split(title, " ")
+	author, _ := b["author"].([]string)
+	for _, a := range author {
+		k = append(k, strings.Split(a, " ")...)
+	}
+	publisher, _ := b["publisher"].(string)
+	k = append(k, strings.Split(publisher, " ")...)
+	subject, _ := b["subject"].([]string)
+	k = append(k, subject...)
+	return
+}

trantor.go

@@ -6,6 +6,7 @@ import (
 	"labix.org/v2/mgo/bson"
 	"log"
 	"net/http"
+	"strings"
 )
 
 type aboutData struct {
@@ -43,8 +44,9 @@ func loginHandler(w http.ResponseWriter, r *http.Request, sess *Session) {
 }
 
 type bookData struct {
-	S    Status
-	Book Book
+	S           Status
+	Book        Book
+	Description []string
 }
 
 func bookHandler(w http.ResponseWriter, r *http.Request, sess *Session) {
@@ -64,6 +66,7 @@ func bookHandler(w http.ResponseWriter, r *http.Request, sess *Session) {
 	}
 	db.IncVisit(id)
 	data.Book = books[0]
+	data.Description = strings.Split(data.Book.Description, "\n")
 	loadTemplate(w, "book", data)
 }