Merge branch 'master' into gridfs

Conflicts:
	database.go
This commit is contained in:
Las Zenow 2013-03-20 00:57:59 +01:00
commit 35fd980692
10 changed files with 612 additions and 63 deletions

7
.gitignore vendored
View file

@ -3,8 +3,5 @@ new/
cover/
trantor
.*.swp
upload/upload
upload/books
upload/cover
upload/new
adduser/adduser
tools/adduser/adduser
tools/update/update

View file

@ -1,22 +1,25 @@
package main
const (
PORT = "8080"
DB_IP = "127.0.0.1"
DB_NAME = "trantor"
BOOKS_COLL = "books"
USERS_COLL = "users"
PASS_SALT = "ImperialLibSalt"
TAGS_DISPLAY = 50
SEARCH_ITEMS_PAGE = 20
NEW_ITEMS_PAGE = 50
TEMPLATE_PATH = "templates/"
BOOKS_PATH = "books/"
COVER_PATH = "cover/"
NEW_PATH = "new/"
CSS_PATH = "css/"
JS_PATH = "js/"
IMG_PATH = "img/"
RESIZE_CMD = "/usr/bin/convert -resize 300 -quality 60 "
RESIZE_THUMB_CMD = "/usr/bin/convert -resize 60 -quality 60 "
PORT = "8080"
DB_IP = "127.0.0.1"
DB_NAME = "trantor"
META_COLL = "meta"
BOOKS_COLL = "books"
TAGS_COLL = "tags"
USERS_COLL = "users"
PASS_SALT = "ImperialLibSalt"
MINUTES_UPDATE_TAGS = 10
TAGS_DISPLAY = 50
SEARCH_ITEMS_PAGE = 20
NEW_ITEMS_PAGE = 50
TEMPLATE_PATH = "templates/"
BOOKS_PATH = "books/"
COVER_PATH = "cover/"
NEW_PATH = "new/"
CSS_PATH = "css/"
JS_PATH = "js/"
IMG_PATH = "img/"
RESIZE_CMD = "/usr/bin/convert -resize 300 -quality 60 "
RESIZE_THUMB_CMD = "/usr/bin/convert -resize 60 -quality 60 "
)

View file

@ -4,7 +4,11 @@ import (
"crypto/md5"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"sort"
"time"
)
const (
META_TYPE_TAGS = "tags updated"
)
var db *DB
@ -35,7 +39,9 @@ type Book struct {
type DB struct {
session *mgo.Session
meta *mgo.Collection
books *mgo.Collection
tags *mgo.Collection
user *mgo.Collection
}
@ -48,7 +54,9 @@ func initDB() *DB {
}
database := d.session.DB(DB_NAME)
d.meta = database.C(META_COLL)
d.books = database.C(BOOKS_COLL)
d.tags = database.C(TAGS_COLL)
d.user = database.C(USERS_COLL)
return d
}
@ -169,25 +177,25 @@ func (d *DB) BookActive(id bson.ObjectId) bool {
return book.Active
}
type tagsList []struct {
Subject string "_id"
Count int "value"
func (d *DB) areTagsOutdated() bool {
var result struct {
Id bson.ObjectId `bson:"_id"`
}
err := d.meta.Find(bson.M{"type": META_TYPE_TAGS}).One(&result)
if err != nil {
return true
}
lastUpdate := result.Id.Time()
return time.Since(lastUpdate).Minutes() > MINUTES_UPDATE_TAGS
}
func (t tagsList) Len() int {
return len(t)
}
func (t tagsList) Less(i, j int) bool {
return t[i].Count > t[j].Count
}
func (t tagsList) Swap(i, j int) {
aux := t[i]
t[i] = t[j]
t[j] = aux
}
func (d *DB) updateTags() error {
_, err := d.meta.RemoveAll(bson.M{"type": META_TYPE_TAGS})
if err != nil {
return err
}
func (d *DB) GetTags() (tagsList, error) {
// TODO: cache the tags
var mr mgo.MapReduce
mr.Map = "function() { " +
"if (this.active) { this.subject.forEach(function(s) { emit(s, 1); }); }" +
@ -197,10 +205,33 @@ func (d *DB) GetTags() (tagsList, error) {
"vals.forEach(function() { count += 1; });" +
"return count;" +
"}"
var result tagsList
_, err := d.books.Find(bson.M{"active": true}).MapReduce(&mr, &result)
if err == nil {
sort.Sort(result)
mr.Out = bson.M{"replace": TAGS_COLL}
_, err = d.books.Find(bson.M{"active": true}).MapReduce(&mr, nil)
if err != nil {
return err
}
return result, err
return d.meta.Insert(bson.M{"type": META_TYPE_TAGS})
}
func (d *DB) GetTags(numTags int) ([]string, error) {
if d.areTagsOutdated() {
err := d.updateTags()
if err != nil {
return nil, err
}
}
var result []struct {
Tag string "_id"
}
err := d.tags.Find(nil).Sort("-value").Limit(numTags).All(&result)
if err != nil {
return nil, err
}
tags := make([]string, len(result))
for i, r := range result {
tags[i] = r.Tag
}
return tags, nil
}

7
tools/README Normal file
View file

@ -0,0 +1,7 @@
Some tools to manage trantor:
- adduser. Used to add users to trantor:
$ adduser myNick
Password:
- update. Update the cover of all the books. It might be outdated.

22
tools/update/config.go Normal file
View file

@ -0,0 +1,22 @@
package main
const (
PORT = "8080"
DB_IP = "127.0.0.1"
DB_NAME = "trantor"
BOOKS_COLL = "books"
NEW_BOOKS_COLL = "new"
USERS_COLL = "users"
PASS_SALT = "ImperialLibSalt"
TAGS_DISPLAY = 50
SEARCH_ITEMS_PAGE = 10
TEMPLATE_PATH = "templates/"
BOOKS_PATH = "books/"
COVER_PATH = "cover/"
NEW_PATH = "new/"
CSS_PATH = "css/"
JS_PATH = "js/"
IMG_PATH = "img/"
RESIZE_CMD = "/usr/bin/convert -resize 300 -quality 60 "
RESIZE_THUMB_CMD = "/usr/bin/convert -resize 60 -quality 60 "
)

214
tools/update/database.go Normal file
View file

@ -0,0 +1,214 @@
package main
import (
"crypto/md5"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"sort"
)
var db *DB
type Book struct {
Id string `bson:"_id"`
Title string
Author []string
Contributor string
Publisher string
Description string
Subject []string
Date string
Lang []string
Type string
Format string
Source string
Relation string
Coverage string
Rights string
Meta string
Path string
Cover string
CoverSmall string
Active bool
Keywords []string
}
type DB struct {
session *mgo.Session
books *mgo.Collection
user *mgo.Collection
}
func initDB() *DB {
var err error
d := new(DB)
d.session, err = mgo.Dial(DB_IP)
if err != nil {
panic(err)
}
d.books = d.session.DB(DB_NAME).C(BOOKS_COLL)
d.user = d.session.DB(DB_NAME).C(USERS_COLL)
return d
}
func (d *DB) Close() {
d.session.Close()
}
func md5Pass(pass string) []byte {
h := md5.New()
hash := h.Sum(([]byte)(PASS_SALT + pass))
return hash
}
func (d *DB) SetPassword(user string, pass string) error {
hash := md5Pass(pass)
return d.user.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}})
}
func (d *DB) UserValid(user string, pass string) bool {
hash := md5Pass(pass)
n, err := d.user.Find(bson.M{"user": user, "pass": hash}).Count()
if err != nil {
return false
}
return n != 0
}
func (d *DB) InsertBook(book interface{}) error {
return d.books.Insert(book)
}
func (d *DB) RemoveBook(id bson.ObjectId) error {
return d.books.Remove(bson.M{"_id": id})
}
func (d *DB) UpdateBook(id bson.ObjectId, data interface{}) error {
return d.books.Update(bson.M{"_id": id}, bson.M{"$set": data})
}
func (d *DB) IncVisit(id bson.ObjectId) error {
return d.books.Update(bson.M{"_id": id}, bson.M{"$inc": bson.M{"VisitsCount": 1}})
}
func (d *DB) IncDownload(path string) error {
return d.books.Update(bson.M{"path": path}, bson.M{"$inc": bson.M{"DownloadCount": 1}})
}
/* optional parameters: length and start index
*
* Returns: list of books, number found and err
*/
func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) {
var start, length int
if len(r) > 0 {
length = r[0]
if len(r) > 1 {
start = r[1]
}
}
q := d.books.Find(query).Sort("-_id")
num, err = q.Count()
if err != nil {
return
}
if start != 0 {
q = q.Skip(start)
}
if length != 0 {
q = q.Limit(length)
}
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
/* Get the most visited books
*/
func (d *DB) GetVisitedBooks(num int) (books []Book, err error) {
var q *mgo.Query
q = d.books.Find(bson.M{"active": true}).Sort("-VisitsCount").Limit(num)
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
/* Get the most downloaded books
*/
func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) {
var q *mgo.Query
q = d.books.Find(bson.M{"active": true}).Sort("-DownloadCount").Limit(num)
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
/* Returns: list of books, number found and err
*/
func (d *DB) GetNewBooks() (books []Book, num int, err error) {
var q *mgo.Query
q = d.books.Find(bson.M{"$nor": []bson.M{{"active": true}}}).Sort("-_id")
num, err = q.Count()
if err != nil {
return
}
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
func (d *DB) BookActive(id bson.ObjectId) bool {
var book Book
err := d.books.Find(bson.M{"_id": id}).One(&book)
if err != nil {
return false
}
return book.Active
}
type tagsList []struct {
Subject string "_id"
Count int "value"
}
func (t tagsList) Len() int {
return len(t)
}
func (t tagsList) Less(i, j int) bool {
return t[i].Count > t[j].Count
}
func (t tagsList) Swap(i, j int) {
aux := t[i]
t[i] = t[j]
t[j] = aux
}
func (d *DB) GetTags() (tagsList, error) {
// TODO: cache the tags
var mr mgo.MapReduce
mr.Map = "function() { " +
"if (this.active) { this.subject.forEach(function(s) { emit(s, 1); }); }" +
"}"
mr.Reduce = "function(tag, vals) { " +
"var count = 0;" +
"vals.forEach(function() { count += 1; });" +
"return count;" +
"}"
var result tagsList
_, err := d.books.Find(nil).MapReduce(&mr, &result)
if err == nil {
sort.Sort(result)
}
return result, err
}

265
tools/update/store.go Normal file
View file

@ -0,0 +1,265 @@
package main
import (
"git.gitorious.org/go-pkg/epub.git"
"io"
"log"
"os"
"os/exec"
"regexp"
"strconv"
"strings"
"unicode/utf8"
)
func ParseFile(path string) (string, error) {
book := map[string]interface{}{}
e, err := epub.Open(NEW_PATH+path, 0)
if err != nil {
return "", err
}
defer e.Close()
title := cleanStr(strings.Join(e.Metadata(epub.EPUB_TITLE), ", "))
book["title"] = title
book["author"] = parseAuthr(e.Metadata(epub.EPUB_CREATOR))
book["contributor"] = cleanStr(strings.Join(e.Metadata(epub.EPUB_CONTRIB), ", "))
book["publisher"] = cleanStr(strings.Join(e.Metadata(epub.EPUB_PUBLISHER), ", "))
book["description"] = parseDescription(e.Metadata(epub.EPUB_DESCRIPTION))
book["subject"] = parseSubject(e.Metadata(epub.EPUB_SUBJECT))
book["date"] = parseDate(e.Metadata(epub.EPUB_DATE))
book["lang"] = e.Metadata(epub.EPUB_LANG)
book["type"] = strings.Join(e.Metadata(epub.EPUB_TYPE), ", ")
book["format"] = strings.Join(e.Metadata(epub.EPUB_FORMAT), ", ")
book["source"] = strings.Join(e.Metadata(epub.EPUB_SOURCE), ", ")
book["relation"] = strings.Join(e.Metadata(epub.EPUB_RELATION), ", ")
book["coverage"] = strings.Join(e.Metadata(epub.EPUB_COVERAGE), ", ")
book["rights"] = strings.Join(e.Metadata(epub.EPUB_RIGHTS), ", ")
book["meta"] = strings.Join(e.Metadata(epub.EPUB_META), ", ")
book["path"] = path
cover, coverSmall := getCover(e, title)
book["cover"] = cover
book["coversmall"] = coverSmall
book["keywords"] = keywords(book)
db.InsertBook(book)
return title, nil
}
func StoreNewFile(name string, file io.Reader) (string, error) {
path := storePath(name)
fw, err := os.Create(NEW_PATH + path)
if err != nil {
return "", err
}
defer fw.Close()
const size = 1024
var n int = size
buff := make([]byte, size)
for n == size {
n, err = file.Read(buff)
fw.Write(buff)
}
return path, nil
}
func StoreBook(book Book) (path string, err error) {
title := book.Title
path = validFileName(BOOKS_PATH, title, ".epub")
oldPath := NEW_PATH + book.Path
r, _ := utf8.DecodeRuneInString(title)
folder := string(r)
if _, err = os.Stat(BOOKS_PATH + folder); err != nil {
err = os.Mkdir(BOOKS_PATH+folder, os.ModePerm)
if err != nil {
log.Println("Error creating", BOOKS_PATH+folder, ":", err.Error())
return
}
}
cmd := exec.Command("mv", oldPath, BOOKS_PATH+path)
err = cmd.Run()
return
}
func DeleteBook(book Book) {
if book.Cover != "" {
os.RemoveAll(book.Cover[1:])
}
if book.CoverSmall != "" {
os.RemoveAll(book.CoverSmall[1:])
}
os.RemoveAll(book.Path)
}
func validFileName(path string, title string, extension string) string {
title = strings.Replace(title, "/", "_", -1)
title = strings.Replace(title, "?", "_", -1)
title = strings.Replace(title, "#", "_", -1)
r, _ := utf8.DecodeRuneInString(title)
folder := string(r)
file := folder + "/" + title + extension
_, err := os.Stat(path + file)
for i := 0; err == nil; i++ {
file = folder + "/" + title + "_" + strconv.Itoa(i) + extension
_, err = os.Stat(path + file)
}
return file
}
func storePath(name string) string {
path := name
_, err := os.Stat(NEW_PATH + path)
for i := 0; err == nil; i++ {
path = strconv.Itoa(i) + "_" + name
_, err = os.Stat(NEW_PATH + path)
}
return path
}
func cleanStr(str string) string {
str = strings.Replace(str, "'", "'", -1)
exp, _ := regexp.Compile("&[^;]*;")
str = exp.ReplaceAllString(str, "")
exp, _ = regexp.Compile("[ ,]*$")
str = exp.ReplaceAllString(str, "")
return str
}
func storeImg(img []byte, title, extension string) (string, string) {
r, _ := utf8.DecodeRuneInString(title)
folder := string(r)
if _, err := os.Stat(COVER_PATH + folder); err != nil {
err = os.Mkdir(COVER_PATH+folder, os.ModePerm)
if err != nil {
log.Println("Error creating", COVER_PATH+folder, ":", err.Error())
return "", ""
}
}
imgPath := validFileName(COVER_PATH, title, extension)
/* store img on disk */
file, err := os.Create(COVER_PATH + imgPath)
if err != nil {
log.Println("Error creating", COVER_PATH+imgPath, ":", err.Error())
return "", ""
}
defer file.Close()
file.Write(img)
/* resize img */
resize := append(strings.Split(RESIZE_CMD, " "), COVER_PATH+imgPath, COVER_PATH+imgPath)
cmd := exec.Command(resize[0], resize[1:]...)
cmd.Run()
imgPathSmall := validFileName(COVER_PATH, title, "_small"+extension)
resize = append(strings.Split(RESIZE_THUMB_CMD, " "), COVER_PATH+imgPath, COVER_PATH+imgPathSmall)
cmd = exec.Command(resize[0], resize[1:]...)
cmd.Run()
return imgPath, imgPathSmall
}
func getCover(e *epub.Epub, title string) (string, string) {
/* Try first common names */
for _, p := range []string{"cover.jpg", "Images/cover.jpg", "cover.jpeg", "cover1.jpg", "cover1.jpeg"} {
img := e.Data(p)
if len(img) != 0 {
return storeImg(img, title, ".jpg")
}
}
/* search for img on the text */
exp, _ := regexp.Compile("<ima?g.*[(src)(href)]=[\"']([^\"']*(\\.[^\\.\"']*))[\"']")
it := e.Iterator(epub.EITERATOR_SPINE)
defer it.Close()
var err error = nil
txt := it.Curr()
for err == nil {
res := exp.FindStringSubmatch(txt)
if res != nil {
urlPart := strings.Split(it.CurrUrl(), "/")
url := strings.Join(urlPart[:len(urlPart)-1], "/")
if res[1][:3] == "../" {
res[1] = res[1][3:]
url = strings.Join(urlPart[:len(urlPart)-2], "/")
}
res[1] = strings.Replace(res[1], "%20", " ", -1)
res[1] = strings.Replace(res[1], "%27", "'", -1)
res[1] = strings.Replace(res[1], "%28", "(", -1)
res[1] = strings.Replace(res[1], "%29", ")", -1)
if url == "" {
url = res[1]
} else {
url = url + "/" + res[1]
}
img := e.Data(url)
if len(img) != 0 {
return storeImg(img, title, res[2])
}
}
txt, err = it.Next()
}
return "", ""
}
func parseAuthr(creator []string) []string {
exp1, _ := regexp.Compile("^(.*\\( *([^\\)]*) *\\))*$")
exp2, _ := regexp.Compile("^[^:]*: *(.*)$")
res := make([]string, len(creator))
for i, s := range creator {
auth := exp1.FindStringSubmatch(s)
if auth != nil {
res[i] = cleanStr(strings.Join(auth[2:], ", "))
} else {
auth := exp2.FindStringSubmatch(s)
if auth != nil {
res[i] = cleanStr(auth[1])
} else {
res[i] = cleanStr(s)
}
}
}
return res
}
func parseDescription(description []string) string {
str := cleanStr(strings.Join(description, ", "))
exp, _ := regexp.Compile("<[^>]*>")
str = exp.ReplaceAllString(str, "")
str = strings.Replace(str, "&amp;", "&", -1)
str = strings.Replace(str, "&lt;", "<", -1)
str = strings.Replace(str, "&gt;", ">", -1)
str = strings.Replace(str, "\\n", "\n", -1)
return str
}
func parseSubject(subject []string) []string {
var res []string
for _, s := range subject {
res = append(res, strings.Split(s, " / ")...)
}
return res
}
func parseDate(date []string) string {
if len(date) == 0 {
return ""
}
return strings.Replace(date[0], "Unspecified: ", "", -1)
}
func keywords(b map[string]interface{}) (k []string) {
title, _ := b["title"].(string)
k = strings.Split(title, " ")
author, _ := b["author"].([]string)
for _, a := range author {
k = append(k, strings.Split(a, " ")...)
}
publisher, _ := b["publisher"].(string)
k = append(k, strings.Split(publisher, " ")...)
subject, _ := b["subject"].([]string)
k = append(k, subject...)
return
}

27
tools/update/update.go Normal file
View file

@ -0,0 +1,27 @@
package main
import (
"fmt"
"git.gitorious.org/go-pkg/epub.git"
"labix.org/v2/mgo/bson"
)
func main() {
db = initDB()
defer db.Close()
books, _, _ := db.GetBooks(bson.M{})
for _, book := range books {
fmt.Println(book.Title)
e, err := epub.Open(BOOKS_PATH+book.Path, 0)
if err != nil {
fmt.Println("================", err)
}
cover, coverSmall := getCover(e, book.Title)
if cover != "" {
db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"cover": cover, "coversmall": coverSmall})
}
e.Close()
}
}

View file

@ -82,24 +82,7 @@ type indexData struct {
func indexHandler(w http.ResponseWriter, r *http.Request) {
var data indexData
/* get the tags */
tags, err := db.GetTags()
if err == nil {
length := len(tags)
if length > TAGS_DISPLAY {
length = TAGS_DISPLAY
}
data.Tags = make([]string, length)
for i, tag := range tags {
if i == TAGS_DISPLAY {
break /* display only 50 */
}
if tag.Subject != "" {
data.Tags[i] = tag.Subject
}
}
}
data.Tags, _ = db.GetTags(TAGS_DISPLAY)
data.S = GetStatus(w, r)
data.S.Home = true
data.Books, data.Count, _ = db.GetBooks(bson.M{"active": true}, 6)