diff --git a/.gitignore b/.gitignore index e2fb1a7..865ec4a 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ trantor tools/adduser/adduser tools/update/update tools/togridfs/togridfs +tools/getISBNnDesc/getISBNnDesc diff --git a/store.go b/store.go index 8e3c5c7..e50e65f 100644 --- a/store.go +++ b/store.go @@ -87,7 +87,8 @@ func parseAuthr(creator []string) []string { } func parseDescription(description []string) string { - str := cleanStr(strings.Join(description, ", ")) + str := cleanStr(strings.Join(description, "\n")) + str = strings.Replace(str, "

", "\n", -1) exp, _ := regexp.Compile("<[^>]*>") str = exp.ReplaceAllString(str, "") str = strings.Replace(str, "&", "&", -1) diff --git a/templates/book.html b/templates/book.html index 02f273f..efb4eed 100644 --- a/templates/book.html +++ b/templates/book.html @@ -60,16 +60,16 @@ function delBook(){ +{{end}}
- {{if .Description}}
-

{{.Description}}

-
+ {{range .Description}} +

{{.}}

{{end}}
+ -{{end}} {{template "footer.html"}} diff --git a/tools/README b/tools/README index 4c16403..cadb0d8 100644 --- a/tools/README +++ b/tools/README @@ -7,3 +7,5 @@ Password: - update. Update the cover of all the books. It might be outdated. - togridfs (23/4/2013). Migrate all files and covers to gridfs + +- getISBNnDesc (31/5/2013). Import the ISBN and the description with changes of lines to the database diff --git a/tools/getISBNnDesc/config.go b/tools/getISBNnDesc/config.go new file mode 100644 index 0000000..c144607 --- /dev/null +++ b/tools/getISBNnDesc/config.go @@ -0,0 +1,32 @@ +package main + +const ( + PORT = "8080" + + DB_IP = "127.0.0.1" + DB_NAME = "trantor" + META_COLL = "meta" + BOOKS_COLL = "books" + TAGS_COLL = "tags" + USERS_COLL = "users" + STATS_COLL = "statistics" + FS_BOOKS = "fs_books" + FS_IMGS = "fs_imgs" + + PASS_SALT = "ImperialLibSalt" + MINUTES_UPDATE_TAGS = 10 + TAGS_DISPLAY = 50 + SEARCH_ITEMS_PAGE = 20 + NEW_ITEMS_PAGE = 50 + + TEMPLATE_PATH = "templates/" + CSS_PATH = "css/" + JS_PATH = "js/" + IMG_PATH = "img/" + + IMG_WIDTH_BIG = 300 + IMG_WIDTH_SMALL = 60 + IMG_QUALITY = 80 + + CHAN_SIZE = 100 +) diff --git a/tools/getISBNnDesc/database.go b/tools/getISBNnDesc/database.go new file mode 100644 index 0000000..32003cc --- /dev/null +++ b/tools/getISBNnDesc/database.go @@ -0,0 +1,248 @@ +package main + +import ( + "crypto/md5" + "labix.org/v2/mgo" + "labix.org/v2/mgo/bson" + "time" +) + +const ( + META_TYPE_TAGS = "tags updated" +) + +var db *DB + +type Book struct { + Id string `bson:"_id"` + Title string + Author []string + Contributor string + Publisher string + Description string + Subject []string + Date string + Lang []string + Isbn string + Type string + Format string + Source string + Relation string + Coverage string + Rights string + Meta string + File bson.ObjectId + Cover bson.ObjectId + CoverSmall bson.ObjectId + Active bool + Keywords []string +} + +type DB struct { + session *mgo.Session + meta *mgo.Collection + books *mgo.Collection + tags *mgo.Collection + user *mgo.Collection + stats *mgo.Collection +} + +func initDB() *DB { + var err error + d := new(DB) + d.session, err = mgo.Dial(DB_IP) + if err != nil { + panic(err) + } + + database := d.session.DB(DB_NAME) + d.meta = database.C(META_COLL) + d.books = database.C(BOOKS_COLL) + d.tags = database.C(TAGS_COLL) + d.user = database.C(USERS_COLL) + d.stats = database.C(STATS_COLL) + return d +} + +func (d *DB) Close() { + d.session.Close() +} + +func md5Pass(pass string) []byte { + h := md5.New() + hash := h.Sum(([]byte)(PASS_SALT + pass)) + return hash +} + +func (d *DB) SetPassword(user string, pass string) error { + hash := md5Pass(pass) + return d.user.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}}) +} + +func (d *DB) UserValid(user string, pass string) bool { + hash := md5Pass(pass) + n, err := d.user.Find(bson.M{"user": user, "pass": hash}).Count() + if err != nil { + return false + } + return n != 0 +} + +func (d *DB) InsertStats(stats interface{}) error { + return d.stats.Insert(stats) +} + +func (d *DB) InsertBook(book interface{}) error { + return d.books.Insert(book) +} + +func (d *DB) RemoveBook(id bson.ObjectId) error { + return d.books.Remove(bson.M{"_id": id}) +} + +func (d *DB) UpdateBook(id bson.ObjectId, data interface{}) error { + return d.books.Update(bson.M{"_id": id}, bson.M{"$set": data}) +} + +func (d *DB) IncVisit(id bson.ObjectId) error { + return d.books.Update(bson.M{"_id": id}, bson.M{"$inc": bson.M{"VisitsCount": 1}}) +} + +func (d *DB) IncDownload(id bson.ObjectId) error { + return d.books.Update(bson.M{"_id": id}, bson.M{"$inc": bson.M{"DownloadCount": 1}}) +} + +/* optional parameters: length and start index + * + * Returns: list of books, number found and err + */ +func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) { + var start, length int + if len(r) > 0 { + length = r[0] + if len(r) > 1 { + start = r[1] + } + } + q := d.books.Find(query).Sort("-_id") + num, err = q.Count() + if err != nil { + return + } + if start != 0 { + q = q.Skip(start) + } + if length != 0 { + q = q.Limit(length) + } + + err = q.All(&books) + for i, b := range books { + books[i].Id = bson.ObjectId(b.Id).Hex() + } + return +} + +/* Get the most visited books + */ +func (d *DB) GetVisitedBooks(num int) (books []Book, err error) { + var q *mgo.Query + q = d.books.Find(bson.M{"active": true}).Sort("-VisitsCount").Limit(num) + err = q.All(&books) + for i, b := range books { + books[i].Id = bson.ObjectId(b.Id).Hex() + } + return +} + +/* Get the most downloaded books + */ +func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) { + var q *mgo.Query + q = d.books.Find(bson.M{"active": true}).Sort("-DownloadCount").Limit(num) + err = q.All(&books) + for i, b := range books { + books[i].Id = bson.ObjectId(b.Id).Hex() + } + return +} + +/* optional parameters: length and start index + * + * Returns: list of books, number found and err + */ +func (d *DB) GetNewBooks(r ...int) (books []Book, num int, err error) { + return d.GetBooks(bson.M{"$nor": []bson.M{{"active": true}}}, r...) +} + +func (d *DB) BookActive(id bson.ObjectId) bool { + var book Book + err := d.books.Find(bson.M{"_id": id}).One(&book) + if err != nil { + return false + } + return book.Active +} + +func (d *DB) GetFS(prefix string) *mgo.GridFS { + return d.session.DB(DB_NAME).GridFS(prefix) +} + +func (d *DB) areTagsOutdated() bool { + var result struct { + Id bson.ObjectId `bson:"_id"` + } + err := d.meta.Find(bson.M{"type": META_TYPE_TAGS}).One(&result) + if err != nil { + return true + } + + lastUpdate := result.Id.Time() + return time.Since(lastUpdate).Minutes() > MINUTES_UPDATE_TAGS +} + +func (d *DB) updateTags() error { + _, err := d.meta.RemoveAll(bson.M{"type": META_TYPE_TAGS}) + if err != nil { + return err + } + + var mr mgo.MapReduce + mr.Map = "function() { " + + "if (this.active) { this.subject.forEach(function(s) { emit(s, 1); }); }" + + "}" + mr.Reduce = "function(tag, vals) { " + + "var count = 0;" + + "vals.forEach(function() { count += 1; });" + + "return count;" + + "}" + mr.Out = bson.M{"replace": TAGS_COLL} + _, err = d.books.Find(bson.M{"active": true}).MapReduce(&mr, nil) + if err != nil { + return err + } + + return d.meta.Insert(bson.M{"type": META_TYPE_TAGS}) +} + +func (d *DB) GetTags(numTags int) ([]string, error) { + if d.areTagsOutdated() { + err := d.updateTags() + if err != nil { + return nil, err + } + } + + var result []struct { + Tag string "_id" + } + err := d.tags.Find(nil).Sort("-value").Limit(numTags).All(&result) + if err != nil { + return nil, err + } + tags := make([]string, len(result)) + for i, r := range result { + tags[i] = r.Tag + } + return tags, nil +} diff --git a/tools/getISBNnDesc/get.go b/tools/getISBNnDesc/get.go new file mode 100644 index 0000000..0875524 --- /dev/null +++ b/tools/getISBNnDesc/get.go @@ -0,0 +1,65 @@ +package main + +import ( + "fmt" + "git.gitorious.org/go-pkg/epubgo.git" + "labix.org/v2/mgo/bson" +) + +func main() { + db = initDB() + defer db.Close() + books, _, _ := db.GetBooks(bson.M{}) + + for _, book := range books { + fmt.Println(book.Title) + e, err := OpenBook(book.File) + if err != nil { + fmt.Println("================", err) + continue + } + + updateISBN(e, book) + updateDescription(e, book) + e.Close() + } +} + +func updateISBN(e *epubgo.Epub, book Book) { + attr, err := e.MetadataAttr("identifier") + if err != nil { + fmt.Println("isbn ================", err) + return + } + data, err := e.Metadata("identifier") + if err != nil { + fmt.Println("isbn ================", err) + return + } + var isbn string + for i, d := range data { + if attr[i]["scheme"] == "ISBN" { + isbn = d + } + } + + if isbn != "" { + db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"isbn": isbn}) + } +} + +func updateDescription(e *epubgo.Epub, book Book) { + descList, err := e.Metadata("description") + if err != nil { + fmt.Println("desc ================", err) + return + } + description := parseDescription(descList) + if len(description) < 10 { + return + } + + if len(book.Description) < 10 || book.Description[:10] == description[:10] { + db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"description": description}) + } +} diff --git a/tools/getISBNnDesc/store.go b/tools/getISBNnDesc/store.go new file mode 100644 index 0000000..e50e65f --- /dev/null +++ b/tools/getISBNnDesc/store.go @@ -0,0 +1,128 @@ +package main + +import ( + "bytes" + "git.gitorious.org/go-pkg/epubgo.git" + "io" + "io/ioutil" + "labix.org/v2/mgo/bson" + "regexp" + "strings" +) + +func OpenBook(id bson.ObjectId) (*epubgo.Epub, error) { + fs := db.GetFS(FS_BOOKS) + f, err := fs.OpenId(id) + if err != nil { + return nil, err + } + defer f.Close() + + buff, err := ioutil.ReadAll(f) + reader := bytes.NewReader(buff) + + return epubgo.Load(reader, int64(len(buff))) +} + +func StoreNewFile(name string, file io.Reader) (bson.ObjectId, error) { + fs := db.GetFS(FS_BOOKS) + fw, err := fs.Create(name) + if err != nil { + return "", err + } + defer fw.Close() + + _, err = io.Copy(fw, file) + id, _ := fw.Id().(bson.ObjectId) + return id, err +} + +func DeleteFile(id bson.ObjectId) error { + fs := db.GetFS(FS_BOOKS) + return fs.RemoveId(id) +} + +func DeleteCover(id bson.ObjectId) error { + fs := db.GetFS(FS_IMGS) + return fs.RemoveId(id) +} + +func DeleteBook(book Book) { + if book.Cover != "" { + DeleteCover(book.Cover) + } + if book.CoverSmall != "" { + DeleteCover(book.CoverSmall) + } + DeleteFile(book.File) +} + +func cleanStr(str string) string { + str = strings.Replace(str, "'", "'", -1) + exp, _ := regexp.Compile("&[^;]*;") + str = exp.ReplaceAllString(str, "") + exp, _ = regexp.Compile("[ ,]*$") + str = exp.ReplaceAllString(str, "") + return str +} + +func parseAuthr(creator []string) []string { + exp1, _ := regexp.Compile("^(.*\\( *([^\\)]*) *\\))*$") + exp2, _ := regexp.Compile("^[^:]*: *(.*)$") + res := make([]string, len(creator)) + for i, s := range creator { + auth := exp1.FindStringSubmatch(s) + if auth != nil { + res[i] = cleanStr(strings.Join(auth[2:], ", ")) + } else { + auth := exp2.FindStringSubmatch(s) + if auth != nil { + res[i] = cleanStr(auth[1]) + } else { + res[i] = cleanStr(s) + } + } + } + return res +} + +func parseDescription(description []string) string { + str := cleanStr(strings.Join(description, "\n")) + str = strings.Replace(str, "

", "\n", -1) + exp, _ := regexp.Compile("<[^>]*>") + str = exp.ReplaceAllString(str, "") + str = strings.Replace(str, "&", "&", -1) + str = strings.Replace(str, "<", "<", -1) + str = strings.Replace(str, ">", ">", -1) + str = strings.Replace(str, "\\n", "\n", -1) + return str +} + +func parseSubject(subject []string) []string { + var res []string + for _, s := range subject { + res = append(res, strings.Split(s, " / ")...) + } + return res +} + +func parseDate(date []string) string { + if len(date) == 0 { + return "" + } + return strings.Replace(date[0], "Unspecified: ", "", -1) +} + +func keywords(b map[string]interface{}) (k []string) { + title, _ := b["title"].(string) + k = strings.Split(title, " ") + author, _ := b["author"].([]string) + for _, a := range author { + k = append(k, strings.Split(a, " ")...) + } + publisher, _ := b["publisher"].(string) + k = append(k, strings.Split(publisher, " ")...) + subject, _ := b["subject"].([]string) + k = append(k, subject...) + return +} diff --git a/trantor.go b/trantor.go index 7f2f6ff..e2b6ed5 100644 --- a/trantor.go +++ b/trantor.go @@ -6,6 +6,7 @@ import ( "labix.org/v2/mgo/bson" "log" "net/http" + "strings" ) type aboutData struct { @@ -43,8 +44,9 @@ func loginHandler(w http.ResponseWriter, r *http.Request, sess *Session) { } type bookData struct { - S Status - Book Book + S Status + Book Book + Description []string } func bookHandler(w http.ResponseWriter, r *http.Request, sess *Session) { @@ -64,6 +66,7 @@ func bookHandler(w http.ResponseWriter, r *http.Request, sess *Session) { } db.IncVisit(id) data.Book = books[0] + data.Description = strings.Split(data.Book.Description, "\n") loadTemplate(w, "book", data) }