diff --git a/.gitignore b/.gitignore index c6c1e7f..5eeb721 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,5 @@ new/ cover/ trantor .*.swp -upload/upload -upload/books -upload/cover -upload/new -adduser/adduser +tools/adduser/adduser +tools/update/update diff --git a/config.go b/config.go index 2dfd90e..44841a4 100644 --- a/config.go +++ b/config.go @@ -1,22 +1,25 @@ package main const ( - PORT = "8080" - DB_IP = "127.0.0.1" - DB_NAME = "trantor" - BOOKS_COLL = "books" - USERS_COLL = "users" - PASS_SALT = "ImperialLibSalt" - TAGS_DISPLAY = 50 - SEARCH_ITEMS_PAGE = 20 - NEW_ITEMS_PAGE = 50 - TEMPLATE_PATH = "templates/" - BOOKS_PATH = "books/" - COVER_PATH = "cover/" - NEW_PATH = "new/" - CSS_PATH = "css/" - JS_PATH = "js/" - IMG_PATH = "img/" - RESIZE_CMD = "/usr/bin/convert -resize 300 -quality 60 " - RESIZE_THUMB_CMD = "/usr/bin/convert -resize 60 -quality 60 " + PORT = "8080" + DB_IP = "127.0.0.1" + DB_NAME = "trantor" + META_COLL = "meta" + BOOKS_COLL = "books" + TAGS_COLL = "tags" + USERS_COLL = "users" + PASS_SALT = "ImperialLibSalt" + MINUTES_UPDATE_TAGS = 10 + TAGS_DISPLAY = 50 + SEARCH_ITEMS_PAGE = 20 + NEW_ITEMS_PAGE = 50 + TEMPLATE_PATH = "templates/" + BOOKS_PATH = "books/" + COVER_PATH = "cover/" + NEW_PATH = "new/" + CSS_PATH = "css/" + JS_PATH = "js/" + IMG_PATH = "img/" + RESIZE_CMD = "/usr/bin/convert -resize 300 -quality 60 " + RESIZE_THUMB_CMD = "/usr/bin/convert -resize 60 -quality 60 " ) diff --git a/database.go b/database.go index f9f29ee..bc00874 100644 --- a/database.go +++ b/database.go @@ -4,7 +4,11 @@ import ( "crypto/md5" "labix.org/v2/mgo" "labix.org/v2/mgo/bson" - "sort" + "time" +) + +const ( + META_TYPE_TAGS = "tags updated" ) var db *DB @@ -35,7 +39,9 @@ type Book struct { type DB struct { session *mgo.Session + meta *mgo.Collection books *mgo.Collection + tags *mgo.Collection user *mgo.Collection } @@ -48,7 +54,9 @@ func initDB() *DB { } database := d.session.DB(DB_NAME) + d.meta = database.C(META_COLL) d.books = database.C(BOOKS_COLL) + d.tags = database.C(TAGS_COLL) d.user = database.C(USERS_COLL) return d } @@ -169,25 +177,25 @@ func (d *DB) BookActive(id bson.ObjectId) bool { return book.Active } -type tagsList []struct { - Subject string "_id" - Count int "value" +func (d *DB) areTagsOutdated() bool { + var result struct { + Id bson.ObjectId `bson:"_id"` + } + err := d.meta.Find(bson.M{"type": META_TYPE_TAGS}).One(&result) + if err != nil { + return true + } + + lastUpdate := result.Id.Time() + return time.Since(lastUpdate).Minutes() > MINUTES_UPDATE_TAGS } -func (t tagsList) Len() int { - return len(t) -} -func (t tagsList) Less(i, j int) bool { - return t[i].Count > t[j].Count -} -func (t tagsList) Swap(i, j int) { - aux := t[i] - t[i] = t[j] - t[j] = aux -} +func (d *DB) updateTags() error { + _, err := d.meta.RemoveAll(bson.M{"type": META_TYPE_TAGS}) + if err != nil { + return err + } -func (d *DB) GetTags() (tagsList, error) { - // TODO: cache the tags var mr mgo.MapReduce mr.Map = "function() { " + "if (this.active) { this.subject.forEach(function(s) { emit(s, 1); }); }" + @@ -197,10 +205,33 @@ func (d *DB) GetTags() (tagsList, error) { "vals.forEach(function() { count += 1; });" + "return count;" + "}" - var result tagsList - _, err := d.books.Find(bson.M{"active": true}).MapReduce(&mr, &result) - if err == nil { - sort.Sort(result) + mr.Out = bson.M{"replace": TAGS_COLL} + _, err = d.books.Find(bson.M{"active": true}).MapReduce(&mr, nil) + if err != nil { + return err } - return result, err + + return d.meta.Insert(bson.M{"type": META_TYPE_TAGS}) +} + +func (d *DB) GetTags(numTags int) ([]string, error) { + if d.areTagsOutdated() { + err := d.updateTags() + if err != nil { + return nil, err + } + } + + var result []struct { + Tag string "_id" + } + err := d.tags.Find(nil).Sort("-value").Limit(numTags).All(&result) + if err != nil { + return nil, err + } + tags := make([]string, len(result)) + for i, r := range result { + tags[i] = r.Tag + } + return tags, nil } diff --git a/tools/README b/tools/README new file mode 100644 index 0000000..0933b15 --- /dev/null +++ b/tools/README @@ -0,0 +1,7 @@ +Some tools to manage trantor: + +- adduser. Used to add users to trantor: +$ adduser myNick +Password: + +- update. Update the cover of all the books. It might be outdated. diff --git a/adduser/adduser.go b/tools/adduser/adduser.go similarity index 100% rename from adduser/adduser.go rename to tools/adduser/adduser.go diff --git a/tools/update/config.go b/tools/update/config.go new file mode 100644 index 0000000..fdbb611 --- /dev/null +++ b/tools/update/config.go @@ -0,0 +1,22 @@ +package main + +const ( + PORT = "8080" + DB_IP = "127.0.0.1" + DB_NAME = "trantor" + BOOKS_COLL = "books" + NEW_BOOKS_COLL = "new" + USERS_COLL = "users" + PASS_SALT = "ImperialLibSalt" + TAGS_DISPLAY = 50 + SEARCH_ITEMS_PAGE = 10 + TEMPLATE_PATH = "templates/" + BOOKS_PATH = "books/" + COVER_PATH = "cover/" + NEW_PATH = "new/" + CSS_PATH = "css/" + JS_PATH = "js/" + IMG_PATH = "img/" + RESIZE_CMD = "/usr/bin/convert -resize 300 -quality 60 " + RESIZE_THUMB_CMD = "/usr/bin/convert -resize 60 -quality 60 " +) diff --git a/tools/update/database.go b/tools/update/database.go new file mode 100644 index 0000000..d2d8527 --- /dev/null +++ b/tools/update/database.go @@ -0,0 +1,214 @@ +package main + +import ( + "crypto/md5" + "labix.org/v2/mgo" + "labix.org/v2/mgo/bson" + "sort" +) + +var db *DB + +type Book struct { + Id string `bson:"_id"` + Title string + Author []string + Contributor string + Publisher string + Description string + Subject []string + Date string + Lang []string + Type string + Format string + Source string + Relation string + Coverage string + Rights string + Meta string + Path string + Cover string + CoverSmall string + Active bool + Keywords []string +} + +type DB struct { + session *mgo.Session + books *mgo.Collection + user *mgo.Collection +} + +func initDB() *DB { + var err error + d := new(DB) + d.session, err = mgo.Dial(DB_IP) + if err != nil { + panic(err) + } + + d.books = d.session.DB(DB_NAME).C(BOOKS_COLL) + d.user = d.session.DB(DB_NAME).C(USERS_COLL) + return d +} + +func (d *DB) Close() { + d.session.Close() +} + +func md5Pass(pass string) []byte { + h := md5.New() + hash := h.Sum(([]byte)(PASS_SALT + pass)) + return hash +} + +func (d *DB) SetPassword(user string, pass string) error { + hash := md5Pass(pass) + return d.user.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}}) +} + +func (d *DB) UserValid(user string, pass string) bool { + hash := md5Pass(pass) + n, err := d.user.Find(bson.M{"user": user, "pass": hash}).Count() + if err != nil { + return false + } + return n != 0 +} + +func (d *DB) InsertBook(book interface{}) error { + return d.books.Insert(book) +} + +func (d *DB) RemoveBook(id bson.ObjectId) error { + return d.books.Remove(bson.M{"_id": id}) +} + +func (d *DB) UpdateBook(id bson.ObjectId, data interface{}) error { + return d.books.Update(bson.M{"_id": id}, bson.M{"$set": data}) +} + +func (d *DB) IncVisit(id bson.ObjectId) error { + return d.books.Update(bson.M{"_id": id}, bson.M{"$inc": bson.M{"VisitsCount": 1}}) +} + +func (d *DB) IncDownload(path string) error { + return d.books.Update(bson.M{"path": path}, bson.M{"$inc": bson.M{"DownloadCount": 1}}) +} + +/* optional parameters: length and start index + * + * Returns: list of books, number found and err + */ +func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) { + var start, length int + if len(r) > 0 { + length = r[0] + if len(r) > 1 { + start = r[1] + } + } + q := d.books.Find(query).Sort("-_id") + num, err = q.Count() + if err != nil { + return + } + if start != 0 { + q = q.Skip(start) + } + if length != 0 { + q = q.Limit(length) + } + + err = q.All(&books) + for i, b := range books { + books[i].Id = bson.ObjectId(b.Id).Hex() + } + return +} + +/* Get the most visited books + */ +func (d *DB) GetVisitedBooks(num int) (books []Book, err error) { + var q *mgo.Query + q = d.books.Find(bson.M{"active": true}).Sort("-VisitsCount").Limit(num) + err = q.All(&books) + for i, b := range books { + books[i].Id = bson.ObjectId(b.Id).Hex() + } + return +} + +/* Get the most downloaded books + */ +func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) { + var q *mgo.Query + q = d.books.Find(bson.M{"active": true}).Sort("-DownloadCount").Limit(num) + err = q.All(&books) + for i, b := range books { + books[i].Id = bson.ObjectId(b.Id).Hex() + } + return +} + +/* Returns: list of books, number found and err + */ +func (d *DB) GetNewBooks() (books []Book, num int, err error) { + var q *mgo.Query + q = d.books.Find(bson.M{"$nor": []bson.M{{"active": true}}}).Sort("-_id") + num, err = q.Count() + if err != nil { + return + } + + err = q.All(&books) + for i, b := range books { + books[i].Id = bson.ObjectId(b.Id).Hex() + } + return +} + +func (d *DB) BookActive(id bson.ObjectId) bool { + var book Book + err := d.books.Find(bson.M{"_id": id}).One(&book) + if err != nil { + return false + } + return book.Active +} + +type tagsList []struct { + Subject string "_id" + Count int "value" +} + +func (t tagsList) Len() int { + return len(t) +} +func (t tagsList) Less(i, j int) bool { + return t[i].Count > t[j].Count +} +func (t tagsList) Swap(i, j int) { + aux := t[i] + t[i] = t[j] + t[j] = aux +} + +func (d *DB) GetTags() (tagsList, error) { + // TODO: cache the tags + var mr mgo.MapReduce + mr.Map = "function() { " + + "if (this.active) { this.subject.forEach(function(s) { emit(s, 1); }); }" + + "}" + mr.Reduce = "function(tag, vals) { " + + "var count = 0;" + + "vals.forEach(function() { count += 1; });" + + "return count;" + + "}" + var result tagsList + _, err := d.books.Find(nil).MapReduce(&mr, &result) + if err == nil { + sort.Sort(result) + } + return result, err +} diff --git a/tools/update/store.go b/tools/update/store.go new file mode 100644 index 0000000..57c0c1e --- /dev/null +++ b/tools/update/store.go @@ -0,0 +1,265 @@ +package main + +import ( + "git.gitorious.org/go-pkg/epub.git" + "io" + "log" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "unicode/utf8" +) + +func ParseFile(path string) (string, error) { + book := map[string]interface{}{} + + e, err := epub.Open(NEW_PATH+path, 0) + if err != nil { + return "", err + } + defer e.Close() + + title := cleanStr(strings.Join(e.Metadata(epub.EPUB_TITLE), ", ")) + book["title"] = title + book["author"] = parseAuthr(e.Metadata(epub.EPUB_CREATOR)) + book["contributor"] = cleanStr(strings.Join(e.Metadata(epub.EPUB_CONTRIB), ", ")) + book["publisher"] = cleanStr(strings.Join(e.Metadata(epub.EPUB_PUBLISHER), ", ")) + book["description"] = parseDescription(e.Metadata(epub.EPUB_DESCRIPTION)) + book["subject"] = parseSubject(e.Metadata(epub.EPUB_SUBJECT)) + book["date"] = parseDate(e.Metadata(epub.EPUB_DATE)) + book["lang"] = e.Metadata(epub.EPUB_LANG) + book["type"] = strings.Join(e.Metadata(epub.EPUB_TYPE), ", ") + book["format"] = strings.Join(e.Metadata(epub.EPUB_FORMAT), ", ") + book["source"] = strings.Join(e.Metadata(epub.EPUB_SOURCE), ", ") + book["relation"] = strings.Join(e.Metadata(epub.EPUB_RELATION), ", ") + book["coverage"] = strings.Join(e.Metadata(epub.EPUB_COVERAGE), ", ") + book["rights"] = strings.Join(e.Metadata(epub.EPUB_RIGHTS), ", ") + book["meta"] = strings.Join(e.Metadata(epub.EPUB_META), ", ") + book["path"] = path + cover, coverSmall := getCover(e, title) + book["cover"] = cover + book["coversmall"] = coverSmall + book["keywords"] = keywords(book) + + db.InsertBook(book) + return title, nil +} + +func StoreNewFile(name string, file io.Reader) (string, error) { + path := storePath(name) + fw, err := os.Create(NEW_PATH + path) + if err != nil { + return "", err + } + defer fw.Close() + + const size = 1024 + var n int = size + buff := make([]byte, size) + for n == size { + n, err = file.Read(buff) + fw.Write(buff) + } + return path, nil +} + +func StoreBook(book Book) (path string, err error) { + title := book.Title + path = validFileName(BOOKS_PATH, title, ".epub") + + oldPath := NEW_PATH + book.Path + r, _ := utf8.DecodeRuneInString(title) + folder := string(r) + if _, err = os.Stat(BOOKS_PATH + folder); err != nil { + err = os.Mkdir(BOOKS_PATH+folder, os.ModePerm) + if err != nil { + log.Println("Error creating", BOOKS_PATH+folder, ":", err.Error()) + return + } + } + cmd := exec.Command("mv", oldPath, BOOKS_PATH+path) + err = cmd.Run() + return +} + +func DeleteBook(book Book) { + if book.Cover != "" { + os.RemoveAll(book.Cover[1:]) + } + if book.CoverSmall != "" { + os.RemoveAll(book.CoverSmall[1:]) + } + os.RemoveAll(book.Path) +} + +func validFileName(path string, title string, extension string) string { + title = strings.Replace(title, "/", "_", -1) + title = strings.Replace(title, "?", "_", -1) + title = strings.Replace(title, "#", "_", -1) + r, _ := utf8.DecodeRuneInString(title) + folder := string(r) + file := folder + "/" + title + extension + _, err := os.Stat(path + file) + for i := 0; err == nil; i++ { + file = folder + "/" + title + "_" + strconv.Itoa(i) + extension + _, err = os.Stat(path + file) + } + return file +} + +func storePath(name string) string { + path := name + _, err := os.Stat(NEW_PATH + path) + for i := 0; err == nil; i++ { + path = strconv.Itoa(i) + "_" + name + _, err = os.Stat(NEW_PATH + path) + } + return path +} + +func cleanStr(str string) string { + str = strings.Replace(str, "'", "'", -1) + exp, _ := regexp.Compile("&[^;]*;") + str = exp.ReplaceAllString(str, "") + exp, _ = regexp.Compile("[ ,]*$") + str = exp.ReplaceAllString(str, "") + return str +} + +func storeImg(img []byte, title, extension string) (string, string) { + r, _ := utf8.DecodeRuneInString(title) + folder := string(r) + if _, err := os.Stat(COVER_PATH + folder); err != nil { + err = os.Mkdir(COVER_PATH+folder, os.ModePerm) + if err != nil { + log.Println("Error creating", COVER_PATH+folder, ":", err.Error()) + return "", "" + } + } + imgPath := validFileName(COVER_PATH, title, extension) + + /* store img on disk */ + file, err := os.Create(COVER_PATH + imgPath) + if err != nil { + log.Println("Error creating", COVER_PATH+imgPath, ":", err.Error()) + return "", "" + } + defer file.Close() + file.Write(img) + + /* resize img */ + resize := append(strings.Split(RESIZE_CMD, " "), COVER_PATH+imgPath, COVER_PATH+imgPath) + cmd := exec.Command(resize[0], resize[1:]...) + cmd.Run() + imgPathSmall := validFileName(COVER_PATH, title, "_small"+extension) + resize = append(strings.Split(RESIZE_THUMB_CMD, " "), COVER_PATH+imgPath, COVER_PATH+imgPathSmall) + cmd = exec.Command(resize[0], resize[1:]...) + cmd.Run() + return imgPath, imgPathSmall +} + +func getCover(e *epub.Epub, title string) (string, string) { + /* Try first common names */ + for _, p := range []string{"cover.jpg", "Images/cover.jpg", "cover.jpeg", "cover1.jpg", "cover1.jpeg"} { + img := e.Data(p) + if len(img) != 0 { + return storeImg(img, title, ".jpg") + } + } + + /* search for img on the text */ + exp, _ := regexp.Compile("]*>") + str = exp.ReplaceAllString(str, "") + str = strings.Replace(str, "&", "&", -1) + str = strings.Replace(str, "<", "<", -1) + str = strings.Replace(str, ">", ">", -1) + str = strings.Replace(str, "\\n", "\n", -1) + return str +} + +func parseSubject(subject []string) []string { + var res []string + for _, s := range subject { + res = append(res, strings.Split(s, " / ")...) + } + return res +} + +func parseDate(date []string) string { + if len(date) == 0 { + return "" + } + return strings.Replace(date[0], "Unspecified: ", "", -1) +} + +func keywords(b map[string]interface{}) (k []string) { + title, _ := b["title"].(string) + k = strings.Split(title, " ") + author, _ := b["author"].([]string) + for _, a := range author { + k = append(k, strings.Split(a, " ")...) + } + publisher, _ := b["publisher"].(string) + k = append(k, strings.Split(publisher, " ")...) + subject, _ := b["subject"].([]string) + k = append(k, subject...) + return +} diff --git a/tools/update/update.go b/tools/update/update.go new file mode 100644 index 0000000..ecde711 --- /dev/null +++ b/tools/update/update.go @@ -0,0 +1,27 @@ +package main + +import ( + "fmt" + "git.gitorious.org/go-pkg/epub.git" + "labix.org/v2/mgo/bson" +) + +func main() { + db = initDB() + defer db.Close() + books, _, _ := db.GetBooks(bson.M{}) + + for _, book := range books { + fmt.Println(book.Title) + e, err := epub.Open(BOOKS_PATH+book.Path, 0) + if err != nil { + fmt.Println("================", err) + } + + cover, coverSmall := getCover(e, book.Title) + if cover != "" { + db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"cover": cover, "coversmall": coverSmall}) + } + e.Close() + } +} diff --git a/trantor.go b/trantor.go index a647dc8..1fcd09b 100644 --- a/trantor.go +++ b/trantor.go @@ -82,24 +82,7 @@ type indexData struct { func indexHandler(w http.ResponseWriter, r *http.Request) { var data indexData - /* get the tags */ - tags, err := db.GetTags() - if err == nil { - length := len(tags) - if length > TAGS_DISPLAY { - length = TAGS_DISPLAY - } - data.Tags = make([]string, length) - for i, tag := range tags { - if i == TAGS_DISPLAY { - break /* display only 50 */ - } - if tag.Subject != "" { - data.Tags[i] = tag.Subject - } - } - } - + data.Tags, _ = db.GetTags(TAGS_DISPLAY) data.S = GetStatus(w, r) data.S.Home = true data.Books, data.Count, _ = db.GetBooks(bson.M{"active": true}, 6)