archives/trantor
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove the tools folder

Browse source 
Everything is moved to it's own repo
This commit is contained in:
Las Zenow 2014-08-28 02:16:10 -05:00
parent c04bbe784e
commit 0cbda42c58
38 changed files with 0 additions and 4999 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
tools/README
View file

@ -1,21 +0,0 @@
Some dirty tools to manage trantor:
- adduser. Used to add users to trantor:
$ adduser myNick
Password:
- importer. import all epubs passed as parameter into the database and approve them
- update. Update the cover of all the books. It might be outdated.
- togridfs (23/4/2013). Migrate all files and covers to gridfs
- getISBNnDesc (31/5/2013). Import the ISBN and the description with changes of lines to the database
- coverNew. Reload the cover from all the new books
- addsize. Add the size of the books to the book metadata
- keywords. Recalculate keywords
- store. Move files from the database to the local storage

38
tools/addsize/addsize.go
View file

@ -1,38 +0,0 @@
package main
import (
"fmt"
"labix.org/v2/mgo/bson"
)
func main() {
db = initDB()
defer db.Close()
books, _, _ := db.GetBooks(bson.M{})
for _, book := range books {
size, err := getSize(book.File)
if err != nil {
fmt.Println(err)
continue
}
err = db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"filesize": size})
if err != nil {
fmt.Println(err)
}
}
}
type file struct {
Length int
}
func getSize(id bson.ObjectId) (int, error) {
fs := db.GetFS(FS_BOOKS)
var f file
err := fs.Find(bson.M{"_id": id}).One(&f)
if err != nil {
return 0, err
}
return f.Length, nil
}

45
tools/addsize/config.go
View file

@ -1,45 +0,0 @@
package main
const (
PORT = "8080"
DB_IP = "127.0.0.1"
DB_NAME = "trantor"
META_COLL = "meta"
BOOKS_COLL = "books"
TAGS_COLL = "tags"
VISITED_COLL = "visited"
DOWNLOADED_COLL = "downloaded"
HOURLY_VISITS_COLL = "visits.hourly"
DAILY_VISITS_COLL = "visits.daily"
MONTHLY_VISITS_COLL = "visits.monthly"
USERS_COLL = "users"
NEWS_COLL = "news"
STATS_COLL = "statistics"
FS_BOOKS = "fs_books"
FS_IMGS = "fs_imgs"
PASS_SALT = "ImperialLibSalt"
MINUTES_UPDATE_TAGS = 11
MINUTES_UPDATE_VISITED = 41
MINUTES_UPDATE_DOWNLOADED = 47
MINUTES_UPDATE_HOURLY = 31
MINUTES_UPDATE_DAILY = 60*12 + 7
MINUTES_UPDATE_MONTHLY = 60*24 + 11
TAGS_DISPLAY = 50
SEARCH_ITEMS_PAGE = 20
NEW_ITEMS_PAGE = 50
NUM_NEWS = 10
DAYS_NEWS_INDEXPAGE = 15
TEMPLATE_PATH = "templates/"
CSS_PATH = "css/"
JS_PATH = "js/"
IMG_PATH = "img/"
IMG_WIDTH_BIG = 300
IMG_WIDTH_SMALL = 60
IMG_QUALITY = 80
CHAN_SIZE = 100
)

243
tools/addsize/database.go
View file

@ -1,243 +0,0 @@
package main
import (
"crypto/md5"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"time"
)
var db *DB
type Book struct {
Id string `bson:"_id"`
Title string
Author []string
Contributor string
Publisher string
Description string
Subject []string
Date string
Lang []string
Isbn string
Type string
Format string
Source string
Relation string
Coverage string
Rights string
Meta string
File bson.ObjectId
FileSize int
Cover bson.ObjectId
CoverSmall bson.ObjectId
Active bool
Keywords []string
}
type News struct {
Date time.Time
Text string
}
type DB struct {
session *mgo.Session
books *mgo.Collection
user *mgo.Collection
news *mgo.Collection
stats *mgo.Collection
mr *MR
}
func initDB() *DB {
var err error
d := new(DB)
d.session, err = mgo.Dial(DB_IP)
if err != nil {
panic(err)
}
database := d.session.DB(DB_NAME)
d.books = database.C(BOOKS_COLL)
d.user = database.C(USERS_COLL)
d.news = database.C(NEWS_COLL)
d.stats = database.C(STATS_COLL)
d.mr = NewMR(database)
return d
}
func (d *DB) Close() {
d.session.Close()
}
func md5Pass(pass string) []byte {
h := md5.New()
hash := h.Sum(([]byte)(PASS_SALT + pass))
return hash
}
func (d *DB) SetPassword(user string, pass string) error {
hash := md5Pass(pass)
return d.user.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}})
}
func (d *DB) UserValid(user string, pass string) bool {
hash := md5Pass(pass)
n, err := d.user.Find(bson.M{"user": user, "pass": hash}).Count()
if err != nil {
return false
}
return n != 0
}
func (d *DB) UserRole(user string) string {
type result struct {
Role string
}
res := result{}
err := d.user.Find(bson.M{"user": user}).One(&res)
if err != nil {
return ""
}
return res.Role
}
func (d *DB) AddNews(text string) error {
var news News
news.Text = text
news.Date = time.Now()
return d.news.Insert(news)
}
func (d *DB) GetNews(num int, days int) (news []News, err error) {
query := bson.M{}
if days != 0 {
duration := time.Duration(-24*days) * time.Hour
date := time.Now().Add(duration)
query = bson.M{"date": bson.M{"$gt": date}}
}
q := d.news.Find(query).Sort("-date").Limit(num)
err = q.All(&news)
return
}
func (d *DB) InsertStats(stats interface{}) error {
return d.stats.Insert(stats)
}
func (d *DB) InsertBook(book interface{}) error {
return d.books.Insert(book)
}
func (d *DB) RemoveBook(id bson.ObjectId) error {
return d.books.Remove(bson.M{"_id": id})
}
func (d *DB) UpdateBook(id bson.ObjectId, data interface{}) error {
return d.books.Update(bson.M{"_id": id}, bson.M{"$set": data})
}
/* optional parameters: length and start index
*
* Returns: list of books, number found and err
*/
func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) {
var start, length int
if len(r) > 0 {
length = r[0]
if len(r) > 1 {
start = r[1]
}
}
q := d.books.Find(query).Sort("-_id")
num, err = q.Count()
if err != nil {
return
}
if start != 0 {
q = q.Skip(start)
}
if length != 0 {
q = q.Limit(length)
}
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
/* Get the most visited books
*/
func (d *DB) GetVisitedBooks(num int) (books []Book, err error) {
bookId, err := d.mr.GetMostVisited(num, d.stats)
if err != nil {
return nil, err
}
books = make([]Book, num)
for i, id := range bookId {
d.books.Find(bson.M{"_id": id}).One(&books[i])
books[i].Id = bson.ObjectId(books[i].Id).Hex()
}
return
}
/* Get the most downloaded books
*/
func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) {
bookId, err := d.mr.GetMostDownloaded(num, d.stats)
if err != nil {
return nil, err
}
books = make([]Book, num)
for i, id := range bookId {
d.books.Find(bson.M{"_id": id}).One(&books[i])
books[i].Id = bson.ObjectId(books[i].Id).Hex()
}
return
}
/* optional parameters: length and start index
*
* Returns: list of books, number found and err
*/
func (d *DB) GetNewBooks(r ...int) (books []Book, num int, err error) {
return d.GetBooks(bson.M{"$nor": []bson.M{{"active": true}}}, r...)
}
func (d *DB) BookActive(id bson.ObjectId) bool {
var book Book
err := d.books.Find(bson.M{"_id": id}).One(&book)
if err != nil {
return false
}
return book.Active
}
func (d *DB) GetFS(prefix string) *mgo.GridFS {
return d.session.DB(DB_NAME).GridFS(prefix)
}
func (d *DB) GetTags(numTags int) ([]string, error) {
return d.mr.GetTags(numTags, d.books)
}
type Visits struct {
Date int64 "_id"
Count int "value"
}
func (d *DB) GetHourVisits(start time.Time) ([]Visits, error) {
return d.mr.GetHourVisits(start, d.stats)
}
func (d *DB) GetDayVisits(start time.Time) ([]Visits, error) {
return d.mr.GetDayVisits(start, d.stats)
}
func (d *DB) GetMonthVisits(start time.Time) ([]Visits, error) {
return d.mr.GetMonthVisits(start, d.stats)
}

266
tools/addsize/mapreduce.go
View file

@ -1,266 +0,0 @@
package main
import (
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"time"
)
type MR struct {
meta *mgo.Collection
tags *mgo.Collection
visited *mgo.Collection
downloaded *mgo.Collection
hourly_raw *mgo.Collection
daily_raw *mgo.Collection
monthly_raw *mgo.Collection
hourly *mgo.Collection
daily *mgo.Collection
monthly *mgo.Collection
}
func NewMR(database *mgo.Database) *MR {
m := new(MR)
m.meta = database.C(META_COLL)
m.tags = database.C(TAGS_COLL)
m.visited = database.C(VISITED_COLL)
m.downloaded = database.C(DOWNLOADED_COLL)
m.hourly_raw = database.C(HOURLY_VISITS_COLL + "_raw")
m.daily_raw = database.C(DAILY_VISITS_COLL + "_raw")
m.monthly_raw = database.C(MONTHLY_VISITS_COLL + "_raw")
m.hourly = database.C(HOURLY_VISITS_COLL)
m.daily = database.C(DAILY_VISITS_COLL)
m.monthly = database.C(MONTHLY_VISITS_COLL)
return m
}
func (m *MR) GetTags(numTags int, booksColl *mgo.Collection) ([]string, error) {
if m.isOutdated(TAGS_COLL, MINUTES_UPDATE_TAGS) {
var mr mgo.MapReduce
mr.Map = `function() {
if (this.subject) {
this.subject.forEach(function(s) { emit(s, 1); });
}
}`
mr.Reduce = `function(tag, vals) {
var count = 0;
vals.forEach(function() { count += 1; });
return count;
}`
err := m.update(&mr, bson.M{"active": true}, booksColl, TAGS_COLL)
if err != nil {
return nil, err
}
}
var result []struct {
Tag string "_id"
}
err := m.tags.Find(nil).Sort("-value").Limit(numTags).All(&result)
if err != nil {
return nil, err
}
tags := make([]string, len(result))
for i, r := range result {
tags[i] = r.Tag
}
return tags, nil
}
func (m *MR) GetMostVisited(num int, statsColl *mgo.Collection) ([]bson.ObjectId, error) {
if m.isOutdated(VISITED_COLL, MINUTES_UPDATE_VISITED) {
var mr mgo.MapReduce
mr.Map = `function() {
emit(this.id, 1);
}`
mr.Reduce = `function(tag, vals) {
var count = 0;
vals.forEach(function() { count += 1; });
return count;
}`
err := m.update(&mr, bson.M{"section": "book"}, statsColl, VISITED_COLL)
if err != nil {
return nil, err
}
}
var result []struct {
Book bson.ObjectId "_id"
}
err := m.visited.Find(nil).Sort("-value").Limit(num).All(&result)
if err != nil {
return nil, err
}
books := make([]bson.ObjectId, len(result))
for i, r := range result {
books[i] = r.Book
}
return books, nil
}
func (m *MR) GetMostDownloaded(num int, statsColl *mgo.Collection) ([]bson.ObjectId, error) {
if m.isOutdated(DOWNLOADED_COLL, MINUTES_UPDATE_DOWNLOADED) {
var mr mgo.MapReduce
mr.Map = `function() {
emit(this.id, 1);
}`
mr.Reduce = `function(tag, vals) {
var count = 0;
vals.forEach(function() { count += 1; });
return count;
}`
err := m.update(&mr, bson.M{"section": "download"}, statsColl, DOWNLOADED_COLL)
if err != nil {
return nil, err
}
}
var result []struct {
Book bson.ObjectId "_id"
}
err := m.downloaded.Find(nil).Sort("-value").Limit(num).All(&result)
if err != nil {
return nil, err
}
books := make([]bson.ObjectId, len(result))
for i, r := range result {
books[i] = r.Book
}
return books, nil
}
func (m *MR) GetHourVisits(start time.Time, statsColl *mgo.Collection) ([]Visits, error) {
if m.isOutdated(HOURLY_VISITS_COLL, MINUTES_UPDATE_HOURLY) {
const reduce = `function(date, vals) {
var count = 0;
vals.forEach(function(v) { count += v; });
return count;
}`
var mr mgo.MapReduce
mr.Map = `function() {
var date = Date.UTC(this.date.getUTCFullYear(),
this.date.getUTCMonth(),
this.date.getUTCDate(),
this.date.getUTCHours());
emit({date: date, session: this.session}, 1);
}`
mr.Reduce = reduce
err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, HOURLY_VISITS_COLL+"_raw")
if err != nil {
return nil, err
}
var mr2 mgo.MapReduce
mr2.Map = `function() {
emit(this['_id']['date'], 1);
}`
mr2.Reduce = reduce
err = m.update(&mr2, bson.M{}, m.hourly_raw, HOURLY_VISITS_COLL)
if err != nil {
return nil, err
}
}
var result []Visits
err := m.hourly.Find(nil).All(&result)
return result, err
}
func (m *MR) GetDayVisits(start time.Time, statsColl *mgo.Collection) ([]Visits, error) {
if m.isOutdated(DAILY_VISITS_COLL, MINUTES_UPDATE_DAILY) {
const reduce = `function(date, vals) {
var count = 0;
vals.forEach(function(v) { count += v; });
return count;
}`
var mr mgo.MapReduce
mr.Map = `function() {
var date = Date.UTC(this.date.getUTCFullYear(),
this.date.getUTCMonth(),
this.date.getUTCDate());
emit({date: date, session: this.session}, 1);
}`
mr.Reduce = reduce
err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, DAILY_VISITS_COLL+"_raw")
if err != nil {
return nil, err
}
var mr2 mgo.MapReduce
mr2.Map = `function() {
emit(this['_id']['date'], 1);
}`
mr2.Reduce = reduce
err = m.update(&mr2, bson.M{}, m.daily_raw, DAILY_VISITS_COLL)
if err != nil {
return nil, err
}
}
var result []Visits
err := m.daily.Find(nil).All(&result)
return result, err
}
func (m *MR) GetMonthVisits(start time.Time, statsColl *mgo.Collection) ([]Visits, error) {
if m.isOutdated(MONTHLY_VISITS_COLL, MINUTES_UPDATE_MONTHLY) {
const reduce = `function(date, vals) {
var count = 0;
vals.forEach(function(v) { count += v; });
return count;
}`
var mr mgo.MapReduce
mr.Map = `function() {
var date = Date.UTC(this.date.getUTCFullYear(),
this.date.getUTCMonth());
emit({date: date, session: this.session}, 1);
}`
mr.Reduce = reduce
err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, MONTHLY_VISITS_COLL+"_raw")
if err != nil {
return nil, err
}
var mr2 mgo.MapReduce
mr2.Map = `function() {
emit(this['_id']['date'], 1);
}`
mr2.Reduce = reduce
err = m.update(&mr2, bson.M{}, m.monthly_raw, MONTHLY_VISITS_COLL)
if err != nil {
return nil, err
}
}
var result []Visits
err := m.monthly.Find(nil).All(&result)
return result, err
}
func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection, storeColl string) error {
_, err := m.meta.RemoveAll(bson.M{"type": storeColl})
if err != nil {
return err
}
mr.Out = bson.M{"replace": storeColl}
_, err = queryColl.Find(query).MapReduce(mr, nil)
if err != nil {
return err
}
return m.meta.Insert(bson.M{"type": storeColl})
}
func (m *MR) isOutdated(coll string, minutes float64) bool {
var result struct {
Id bson.ObjectId `bson:"_id"`
}
err := m.meta.Find(bson.M{"type": coll}).One(&result)
if err != nil {
return true
}
lastUpdate := result.Id.Time()
return time.Since(lastUpdate).Minutes() > minutes
}

28
tools/adduser/adduser.go
View file

@ -1,28 +0,0 @@
package main
import (
"code.google.com/p/gopass"
"git.gitorious.org/trantor/trantor.git/database"
"os"
)
const (
DB_IP = "127.0.0.1"
DB_NAME = "trantor"
)
func main() {
db := database.Init(DB_IP, DB_NAME)
defer db.Close()
user := os.Args[1]
pass, err := gopass.GetPass("Password: ")
if err != nil {
panic(err)
}
err = db.AddUser(user, pass)
if err != nil {
panic(err)
}
}

45
tools/coverNew/config.go
View file

@ -1,45 +0,0 @@
package main
const (
PORT = "8080"
DB_IP = "127.0.0.1"
DB_NAME = "trantor"
META_COLL = "meta"
BOOKS_COLL = "books"
TAGS_COLL = "tags"
VISITED_COLL = "visited"
DOWNLOADED_COLL = "downloaded"
HOURLY_VISITS_COLL = "visits.hourly"
DAILY_VISITS_COLL = "visits.daily"
MONTHLY_VISITS_COLL = "visits.monthly"
USERS_COLL = "users"
NEWS_COLL = "news"
STATS_COLL = "statistics"
FS_BOOKS = "fs_books"
FS_IMGS = "fs_imgs"
PASS_SALT = "ImperialLibSalt"
MINUTES_UPDATE_TAGS = 11
MINUTES_UPDATE_VISITED = 41
MINUTES_UPDATE_DOWNLOADED = 47
MINUTES_UPDATE_HOURLY = 31
MINUTES_UPDATE_DAILY = 60*12 + 7
MINUTES_UPDATE_MONTHLY = 60*24 + 11
TAGS_DISPLAY = 50
SEARCH_ITEMS_PAGE = 20
NEW_ITEMS_PAGE = 50
NUM_NEWS = 10
DAYS_NEWS_INDEXPAGE = 15
TEMPLATE_PATH = "templates/"
CSS_PATH = "css/"
JS_PATH = "js/"
IMG_PATH = "img/"
IMG_WIDTH_BIG = 300
IMG_WIDTH_SMALL = 60
IMG_QUALITY = 80
CHAN_SIZE = 100
)

204
tools/coverNew/cover.go
View file

@ -1,204 +0,0 @@
package main
import _ "image/png"
import _ "image/jpeg"
import _ "image/gif"
import (
"bytes"
"git.gitorious.org/go-pkg/epubgo.git"
"github.com/gorilla/mux"
"github.com/nfnt/resize"
"image"
"image/jpeg"
"io"
"io/ioutil"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"log"
"net/http"
"regexp"
"strings"
)
func coverHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
if !bson.IsObjectIdHex(vars["id"]) {
notFound(w, r)
return
}
id := bson.ObjectIdHex(vars["id"])
books, _, err := db.GetBooks(bson.M{"_id": id})
if err != nil || len(books) == 0 {
notFound(w, r)
return
}
book := books[0]
if !book.Active {
sess := GetSession(r)
if sess.User == "" {
notFound(w, r)
return
}
}
fs := db.GetFS(FS_IMGS)
var f *mgo.GridFile
if vars["size"] == "small" {
f, err = fs.OpenId(book.CoverSmall)
} else {
f, err = fs.OpenId(book.Cover)
}
if err != nil {
log.Println("Error while opening image:", err)
notFound(w, r)
return
}
defer f.Close()
headers := w.Header()
headers["Content-Type"] = []string{"image/jpeg"}
io.Copy(w, f)
}
func GetCover(e *epubgo.Epub, title string) (bson.ObjectId, bson.ObjectId) {
imgId, smallId := coverFromMetadata(e, title)
if imgId != "" {
return imgId, smallId
}
imgId, smallId = searchCommonCoverNames(e, title)
if imgId != "" {
return imgId, smallId
}
/* search for img on the text */
exp, _ := regexp.Compile("<.*ima?g.*[(src)(href)]=[\"']([^\"']*(\\.[^\\.\"']*))[\"']")
it, errNext := e.Spine()
for errNext == nil {
file, err := it.Open()
if err != nil {
break
}
defer file.Close()
txt, err := ioutil.ReadAll(file)
if err != nil {
break
}
res := exp.FindSubmatch(txt)
if res != nil {
href := string(res[1])
urlPart := strings.Split(it.URL(), "/")
url := strings.Join(urlPart[:len(urlPart)-1], "/")
if href[:3] == "../" {
href = href[3:]
url = strings.Join(urlPart[:len(urlPart)-2], "/")
}
href = strings.Replace(href, "%20", " ", -1)
href = strings.Replace(href, "%27", "'", -1)
href = strings.Replace(href, "%28", "(", -1)
href = strings.Replace(href, "%29", ")", -1)
if url == "" {
url = href
} else {
url = url + "/" + href
}
img, err := e.OpenFile(url)
if err == nil {
defer img.Close()
return storeImg(img, title)
}
}
errNext = it.Next()
}
return "", ""
}
func coverFromMetadata(e *epubgo.Epub, title string) (bson.ObjectId, bson.ObjectId) {
metaList, _ := e.MetadataAttr("meta")
for _, meta := range metaList {
if meta["name"] == "cover" {
img, err := e.OpenFileId(meta["content"])
if err == nil {
defer img.Close()
return storeImg(img, title)
}
}
}
return "", ""
}
func searchCommonCoverNames(e *epubgo.Epub, title string) (bson.ObjectId, bson.ObjectId) {
for _, p := range []string{"cover.jpg", "Images/cover.jpg", "images/cover.jpg", "cover.jpeg", "cover1.jpg", "cover1.jpeg"} {
img, err := e.OpenFile(p)
if err == nil {
defer img.Close()
return storeImg(img, title)
}
}
return "", ""
}
func storeImg(img io.Reader, title string) (bson.ObjectId, bson.ObjectId) {
/* open the files */
fBig, err := createCoverFile(title)
if err != nil {
log.Println("Error creating", title, ":", err.Error())
return "", ""
}
defer fBig.Close()
fSmall, err := createCoverFile(title + "_small")
if err != nil {
log.Println("Error creating", title+"_small", ":", err.Error())
return "", ""
}
defer fSmall.Close()
/* resize img */
var img2 bytes.Buffer
img1 := io.TeeReader(img, &img2)
jpgOptions := jpeg.Options{IMG_QUALITY}
imgResized, err := resizeImg(img1, IMG_WIDTH_BIG)
if err != nil {
log.Println("Error resizing big image:", err.Error())
return "", ""
}
err = jpeg.Encode(fBig, imgResized, &jpgOptions)
if err != nil {
log.Println("Error encoding big image:", err.Error())
return "", ""
}
imgSmallResized, err := resizeImg(&img2, IMG_WIDTH_SMALL)
if err != nil {
log.Println("Error resizing small image:", err.Error())
return "", ""
}
err = jpeg.Encode(fSmall, imgSmallResized, &jpgOptions)
if err != nil {
log.Println("Error encoding small image:", err.Error())
return "", ""
}
idBig, _ := fBig.Id().(bson.ObjectId)
idSmall, _ := fSmall.Id().(bson.ObjectId)
return idBig, idSmall
}
func createCoverFile(title string) (*mgo.GridFile, error) {
fs := db.GetFS(FS_IMGS)
return fs.Create(title + ".jpg")
}
func resizeImg(imgReader io.Reader, width uint) (image.Image, error) {
img, _, err := image.Decode(imgReader)
if err != nil {
return nil, err
}
return resize.Resize(width, 0, img, resize.NearestNeighbor), nil
}

31
tools/coverNew/coverNew.go
View file

@ -1,31 +0,0 @@
package main
import (
"fmt"
"labix.org/v2/mgo/bson"
"net/http"
)
func main() {
db = initDB()
defer db.Close()
books, _, _ := db.GetNewBooks()
for _, book := range books {
fmt.Println(book.Title)
e, err := OpenBook(book.File)
if err != nil {
fmt.Println("================", err)
}
cover, coverSmall := GetCover(e, book.Title)
if cover != "" {
db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"cover": cover, "coversmall": coverSmall})
}
e.Close()
}
}
func notFound(w http.ResponseWriter, r *http.Request) {
// cover.go needs this function to compile
}

230
tools/coverNew/database.go
View file

@ -1,230 +0,0 @@
package main
import (
"crypto/md5"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"time"
)
var db *DB
type Book struct {
Id string `bson:"_id"`
Title string
Author []string
Contributor string
Publisher string
Description string
Subject []string
Date string
Lang []string
Isbn string
Type string
Format string
Source string
Relation string
Coverage string
Rights string
Meta string
File bson.ObjectId
Cover bson.ObjectId
CoverSmall bson.ObjectId
Active bool
Keywords []string
}
type News struct {
Date time.Time
Text string
}
type DB struct {
session *mgo.Session
books *mgo.Collection
user *mgo.Collection
news *mgo.Collection
stats *mgo.Collection
mr *MR
}
func initDB() *DB {
var err error
d := new(DB)
d.session, err = mgo.Dial(DB_IP)
if err != nil {
panic(err)
}
database := d.session.DB(DB_NAME)
d.books = database.C(BOOKS_COLL)
d.user = database.C(USERS_COLL)
d.news = database.C(NEWS_COLL)
d.stats = database.C(STATS_COLL)
d.mr = NewMR(database)
return d
}
func (d *DB) Close() {
d.session.Close()
}
func md5Pass(pass string) []byte {
h := md5.New()
hash := h.Sum(([]byte)(PASS_SALT + pass))
return hash
}
func (d *DB) SetPassword(user string, pass string) error {
hash := md5Pass(pass)
return d.user.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}})
}
func (d *DB) UserValid(user string, pass string) bool {
hash := md5Pass(pass)
n, err := d.user.Find(bson.M{"user": user, "pass": hash}).Count()
if err != nil {
return false
}
return n != 0
}
func (d *DB) AddNews(text string) error {
var news News
news.Text = text
news.Date = time.Now()
return d.news.Insert(news)
}
func (d *DB) GetNews(num int, days int) (news []News, err error) {
query := bson.M{}
if days != 0 {
duration := time.Duration(-24*days) * time.Hour
date := time.Now().Add(duration)
query = bson.M{"date": bson.M{"$gt": date}}
}
q := d.news.Find(query).Sort("-date").Limit(num)
err = q.All(&news)
return
}
func (d *DB) InsertStats(stats interface{}) error {
return d.stats.Insert(stats)
}
func (d *DB) InsertBook(book interface{}) error {
return d.books.Insert(book)
}
func (d *DB) RemoveBook(id bson.ObjectId) error {
return d.books.Remove(bson.M{"_id": id})
}
func (d *DB) UpdateBook(id bson.ObjectId, data interface{}) error {
return d.books.Update(bson.M{"_id": id}, bson.M{"$set": data})
}
/* optional parameters: length and start index
*
* Returns: list of books, number found and err
*/
func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) {
var start, length int
if len(r) > 0 {
length = r[0]
if len(r) > 1 {
start = r[1]
}
}
q := d.books.Find(query).Sort("-_id")
num, err = q.Count()
if err != nil {
return
}
if start != 0 {
q = q.Skip(start)
}
if length != 0 {
q = q.Limit(length)
}
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
/* Get the most visited books
*/
func (d *DB) GetVisitedBooks(num int) (books []Book, err error) {
bookId, err := d.mr.GetMostVisited(num, d.stats)
if err != nil {
return nil, err
}
books = make([]Book, num)
for i, id := range bookId {
d.books.Find(bson.M{"_id": id}).One(&books[i])
books[i].Id = bson.ObjectId(books[i].Id).Hex()
}
return
}
/* Get the most downloaded books
*/
func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) {
bookId, err := d.mr.GetMostDownloaded(num, d.stats)
if err != nil {
return nil, err
}
books = make([]Book, num)
for i, id := range bookId {
d.books.Find(bson.M{"_id": id}).One(&books[i])
books[i].Id = bson.ObjectId(books[i].Id).Hex()
}
return
}
/* optional parameters: length and start index
*
* Returns: list of books, number found and err
*/
func (d *DB) GetNewBooks(r ...int) (books []Book, num int, err error) {
return d.GetBooks(bson.M{"$nor": []bson.M{{"active": true}}}, r...)
}
func (d *DB) BookActive(id bson.ObjectId) bool {
var book Book
err := d.books.Find(bson.M{"_id": id}).One(&book)
if err != nil {
return false
}
return book.Active
}
func (d *DB) GetFS(prefix string) *mgo.GridFS {
return d.session.DB(DB_NAME).GridFS(prefix)
}
func (d *DB) GetTags(numTags int) ([]string, error) {
return d.mr.GetTags(numTags, d.books)
}
type Visits struct {
Date int64 "_id"
Count int "value"
}
func (d *DB) GetHourVisits(start time.Time) ([]Visits, error) {
return d.mr.GetHourVisits(start, d.stats)
}
func (d *DB) GetDayVisits(start time.Time) ([]Visits, error) {
return d.mr.GetDayVisits(start, d.stats)
}
func (d *DB) GetMonthVisits(start time.Time) ([]Visits, error) {
return d.mr.GetMonthVisits(start, d.stats)
}

266
tools/coverNew/mapreduce.go
View file

@ -1,266 +0,0 @@
package main
import (
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"time"
)
type MR struct {
meta *mgo.Collection
tags *mgo.Collection
visited *mgo.Collection
downloaded *mgo.Collection
hourly_raw *mgo.Collection
daily_raw *mgo.Collection
monthly_raw *mgo.Collection
hourly *mgo.Collection
daily *mgo.Collection
monthly *mgo.Collection
}
func NewMR(database *mgo.Database) *MR {
m := new(MR)
m.meta = database.C(META_COLL)
m.tags = database.C(TAGS_COLL)
m.visited = database.C(VISITED_COLL)
m.downloaded = database.C(DOWNLOADED_COLL)
m.hourly_raw = database.C(HOURLY_VISITS_COLL + "_raw")
m.daily_raw = database.C(DAILY_VISITS_COLL + "_raw")
m.monthly_raw = database.C(MONTHLY_VISITS_COLL + "_raw")
m.hourly = database.C(HOURLY_VISITS_COLL)
m.daily = database.C(DAILY_VISITS_COLL)
m.monthly = database.C(MONTHLY_VISITS_COLL)
return m
}
func (m *MR) GetTags(numTags int, booksColl *mgo.Collection) ([]string, error) {
if m.isOutdated(TAGS_COLL, MINUTES_UPDATE_TAGS) {
var mr mgo.MapReduce
mr.Map = `function() {
if (this.subject) {
this.subject.forEach(function(s) { emit(s, 1); });
}
}`
mr.Reduce = `function(tag, vals) {
var count = 0;
vals.forEach(function() { count += 1; });
return count;
}`
err := m.update(&mr, bson.M{"active": true}, booksColl, TAGS_COLL)
if err != nil {
return nil, err
}
}
var result []struct {
Tag string "_id"
}
err := m.tags.Find(nil).Sort("-value").Limit(numTags).All(&result)
if err != nil {
return nil, err
}
tags := make([]string, len(result))
for i, r := range result {
tags[i] = r.Tag
}
return tags, nil
}
func (m *MR) GetMostVisited(num int, statsColl *mgo.Collection) ([]bson.ObjectId, error) {
if m.isOutdated(VISITED_COLL, MINUTES_UPDATE_VISITED) {
var mr mgo.MapReduce
mr.Map = `function() {
emit(this.id, 1);
}`
mr.Reduce = `function(tag, vals) {
var count = 0;
vals.forEach(function() { count += 1; });
return count;
}`
err := m.update(&mr, bson.M{"section": "book"}, statsColl, VISITED_COLL)
if err != nil {
return nil, err
}
}
var result []struct {
Book bson.ObjectId "_id"
}
err := m.visited.Find(nil).Sort("-value").Limit(num).All(&result)
if err != nil {
return nil, err
}
books := make([]bson.ObjectId, len(result))
for i, r := range result {
books[i] = r.Book
}
return books, nil
}
func (m *MR) GetMostDownloaded(num int, statsColl *mgo.Collection) ([]bson.ObjectId, error) {
if m.isOutdated(DOWNLOADED_COLL, MINUTES_UPDATE_DOWNLOADED) {
var mr mgo.MapReduce
mr.Map = `function() {
emit(this.id, 1);
}`
mr.Reduce = `function(tag, vals) {
var count = 0;
vals.forEach(function() { count += 1; });
return count;
}`
err := m.update(&mr, bson.M{"section": "download"}, statsColl, DOWNLOADED_COLL)
if err != nil {
return nil, err
}
}
var result []struct {
Book bson.ObjectId "_id"
}
err := m.downloaded.Find(nil).Sort("-value").Limit(num).All(&result)
if err != nil {
return nil, err
}
books := make([]bson.ObjectId, len(result))
for i, r := range result {
books[i] = r.Book
}
return books, nil
}
func (m *MR) GetHourVisits(start time.Time, statsColl *mgo.Collection) ([]Visits, error) {
if m.isOutdated(HOURLY_VISITS_COLL, MINUTES_UPDATE_HOURLY) {
const reduce = `function(date, vals) {
var count = 0;
vals.forEach(function(v) { count += v; });
return count;
}`
var mr mgo.MapReduce
mr.Map = `function() {
var date = Date.UTC(this.date.getUTCFullYear(),
this.date.getUTCMonth(),
this.date.getUTCDate(),
this.date.getUTCHours());
emit({date: date, session: this.session}, 1);
}`
mr.Reduce = reduce
err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, HOURLY_VISITS_COLL+"_raw")
if err != nil {
return nil, err
}
var mr2 mgo.MapReduce
mr2.Map = `function() {
emit(this['_id']['date'], 1);
}`
mr2.Reduce = reduce
err = m.update(&mr2, bson.M{}, m.hourly_raw, HOURLY_VISITS_COLL)
if err != nil {
return nil, err
}
}
var result []Visits
err := m.hourly.Find(nil).All(&result)
return result, err
}
func (m *MR) GetDayVisits(start time.Time, statsColl *mgo.Collection) ([]Visits, error) {
if m.isOutdated(DAILY_VISITS_COLL, MINUTES_UPDATE_DAILY) {
const reduce = `function(date, vals) {
var count = 0;
vals.forEach(function(v) { count += v; });
return count;
}`
var mr mgo.MapReduce
mr.Map = `function() {
var date = Date.UTC(this.date.getUTCFullYear(),
this.date.getUTCMonth(),
this.date.getUTCDate());
emit({date: date, session: this.session}, 1);
}`
mr.Reduce = reduce
err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, DAILY_VISITS_COLL+"_raw")
if err != nil {
return nil, err
}
var mr2 mgo.MapReduce
mr2.Map = `function() {
emit(this['_id']['date'], 1);
}`
mr2.Reduce = reduce
err = m.update(&mr2, bson.M{}, m.daily_raw, DAILY_VISITS_COLL)
if err != nil {
return nil, err
}
}
var result []Visits
err := m.daily.Find(nil).All(&result)
return result, err
}
func (m *MR) GetMonthVisits(start time.Time, statsColl *mgo.Collection) ([]Visits, error) {
if m.isOutdated(MONTHLY_VISITS_COLL, MINUTES_UPDATE_MONTHLY) {
const reduce = `function(date, vals) {
var count = 0;
vals.forEach(function(v) { count += v; });
return count;
}`
var mr mgo.MapReduce
mr.Map = `function() {
var date = Date.UTC(this.date.getUTCFullYear(),
this.date.getUTCMonth());
emit({date: date, session: this.session}, 1);
}`
mr.Reduce = reduce
err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, MONTHLY_VISITS_COLL+"_raw")
if err != nil {
return nil, err
}
var mr2 mgo.MapReduce
mr2.Map = `function() {
emit(this['_id']['date'], 1);
}`
mr2.Reduce = reduce
err = m.update(&mr2, bson.M{}, m.monthly_raw, MONTHLY_VISITS_COLL)
if err != nil {
return nil, err
}
}
var result []Visits
err := m.monthly.Find(nil).All(&result)
return result, err
}
func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection, storeColl string) error {
_, err := m.meta.RemoveAll(bson.M{"type": storeColl})
if err != nil {
return err
}
mr.Out = bson.M{"replace": storeColl}
_, err = queryColl.Find(query).MapReduce(mr, nil)
if err != nil {
return err
}
return m.meta.Insert(bson.M{"type": storeColl})
}
func (m *MR) isOutdated(coll string, minutes float64) bool {
var result struct {
Id bson.ObjectId `bson:"_id"`
}
err := m.meta.Find(bson.M{"type": coll}).One(&result)
if err != nil {
return true
}
lastUpdate := result.Id.Time()
return time.Since(lastUpdate).Minutes() > minutes
}

76
tools/coverNew/session.go
View file

@ -1,76 +0,0 @@
package main
import (
"encoding/hex"
"github.com/gorilla/securecookie"
"github.com/gorilla/sessions"
"net/http"
)
var sesStore = sessions.NewCookieStore(securecookie.GenerateRandomKey(64))
type Notification struct {
Title string
Msg string
Type string /* error, info or success */
}
type Session struct {
User string
Notif []Notification
S *sessions.Session
}
func getNotif(session *sessions.Session) []Notification {
msgs := session.Flashes("nMsg")
titles := session.Flashes("nTitle")
tpes := session.Flashes("nType")
notif := make([]Notification, len(msgs))
for i, m := range msgs {
msg, _ := m.(string)
title, _ := titles[i].(string)
tpe, _ := tpes[i].(string)
notif[i] = Notification{title, msg, tpe}
}
return notif
}
func GetSession(r *http.Request) (s *Session) {
s = new(Session)
var err error
s.S, err = sesStore.Get(r, "session")
if err == nil && !s.S.IsNew {
s.User, _ = s.S.Values["user"].(string)
s.Notif = getNotif(s.S)
}
if s.S.IsNew {
s.S.Values["id"] = hex.EncodeToString(securecookie.GenerateRandomKey(16))
}
return
}
func (s *Session) LogIn(user string) {
s.User = user
s.S.Values["user"] = user
}
func (s *Session) LogOut() {
s.S.Values["user"] = ""
}
func (s *Session) Notify(title, msg, tpe string) {
s.S.AddFlash(msg, "nMsg")
s.S.AddFlash(title, "nTitle")
s.S.AddFlash(tpe, "nType")
}
func (s *Session) Save(w http.ResponseWriter, r *http.Request) {
sesStore.Save(r, w, s.S)
}
func (s *Session) Id() string {
id, _ := s.S.Values["id"].(string)
return id
}

128
tools/coverNew/store.go
View file

@ -1,128 +0,0 @@
package main
import (
"bytes"
"git.gitorious.org/go-pkg/epubgo.git"
"io"
"io/ioutil"
"labix.org/v2/mgo/bson"
"regexp"
"strings"
)
func OpenBook(id bson.ObjectId) (*epubgo.Epub, error) {
fs := db.GetFS(FS_BOOKS)
f, err := fs.OpenId(id)
if err != nil {
return nil, err
}
defer f.Close()
buff, err := ioutil.ReadAll(f)
reader := bytes.NewReader(buff)
return epubgo.Load(reader, int64(len(buff)))
}
func StoreNewFile(name string, file io.Reader) (bson.ObjectId, error) {
fs := db.GetFS(FS_BOOKS)
fw, err := fs.Create(name)
if err != nil {
return "", err
}
defer fw.Close()
_, err = io.Copy(fw, file)
id, _ := fw.Id().(bson.ObjectId)
return id, err
}
func DeleteFile(id bson.ObjectId) error {
fs := db.GetFS(FS_BOOKS)
return fs.RemoveId(id)
}
func DeleteCover(id bson.ObjectId) error {
fs := db.GetFS(FS_IMGS)
return fs.RemoveId(id)
}
func DeleteBook(book Book) {
if book.Cover != "" {
DeleteCover(book.Cover)
}
if book.CoverSmall != "" {
DeleteCover(book.CoverSmall)
}
DeleteFile(book.File)
}
func cleanStr(str string) string {
str = strings.Replace(str, "&#39;", "'", -1)
exp, _ := regexp.Compile("&[^;]*;")
str = exp.ReplaceAllString(str, "")
exp, _ = regexp.Compile("[ ,]*$")
str = exp.ReplaceAllString(str, "")
return str
}
func parseAuthr(creator []string) []string {
exp1, _ := regexp.Compile("^(.*\\( *([^\\)]*) *\\))*$")
exp2, _ := regexp.Compile("^[^:]*: *(.*)$")
res := make([]string, len(creator))
for i, s := range creator {
auth := exp1.FindStringSubmatch(s)
if auth != nil {
res[i] = cleanStr(strings.Join(auth[2:], ", "))
} else {
auth := exp2.FindStringSubmatch(s)
if auth != nil {
res[i] = cleanStr(auth[1])
} else {
res[i] = cleanStr(s)
}
}
}
return res
}
func parseDescription(description []string) string {
str := cleanStr(strings.Join(description, "\n"))
str = strings.Replace(str, "</p>", "\n", -1)
exp, _ := regexp.Compile("<[^>]*>")
str = exp.ReplaceAllString(str, "")
str = strings.Replace(str, "&amp;", "&", -1)
str = strings.Replace(str, "&lt;", "<", -1)
str = strings.Replace(str, "&gt;", ">", -1)
str = strings.Replace(str, "\\n", "\n", -1)
return str
}
func parseSubject(subject []string) []string {
var res []string
for _, s := range subject {
res = append(res, strings.Split(s, " / ")...)
}
return res
}
func parseDate(date []string) string {
if len(date) == 0 {
return ""
}
return strings.Replace(date[0], "Unspecified: ", "", -1)
}
func keywords(b map[string]interface{}) (k []string) {
title, _ := b["title"].(string)
k = strings.Split(title, " ")
author, _ := b["author"].([]string)
for _, a := range author {
k = append(k, strings.Split(a, " ")...)
}
publisher, _ := b["publisher"].(string)
k = append(k, strings.Split(publisher, " ")...)
subject, _ := b["subject"].([]string)
k = append(k, subject...)
return
}

32
tools/getISBNnDesc/config.go
View file

@ -1,32 +0,0 @@
package main
const (
PORT = "8080"
DB_IP = "127.0.0.1"
DB_NAME = "trantor"
META_COLL = "meta"
BOOKS_COLL = "books"
TAGS_COLL = "tags"
USERS_COLL = "users"
STATS_COLL = "statistics"
FS_BOOKS = "fs_books"
FS_IMGS = "fs_imgs"
PASS_SALT = "ImperialLibSalt"
MINUTES_UPDATE_TAGS = 10
TAGS_DISPLAY = 50
SEARCH_ITEMS_PAGE = 20
NEW_ITEMS_PAGE = 50
TEMPLATE_PATH = "templates/"
CSS_PATH = "css/"
JS_PATH = "js/"
IMG_PATH = "img/"
IMG_WIDTH_BIG = 300
IMG_WIDTH_SMALL = 60
IMG_QUALITY = 80
CHAN_SIZE = 100
)

248
tools/getISBNnDesc/database.go
View file

@ -1,248 +0,0 @@
package main
import (
"crypto/md5"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"time"
)
const (
META_TYPE_TAGS = "tags updated"
)
var db *DB
type Book struct {
Id string `bson:"_id"`
Title string
Author []string
Contributor string
Publisher string
Description string
Subject []string
Date string
Lang []string
Isbn string
Type string
Format string
Source string
Relation string
Coverage string
Rights string
Meta string
File bson.ObjectId
Cover bson.ObjectId
CoverSmall bson.ObjectId
Active bool
Keywords []string
}
type DB struct {
session *mgo.Session
meta *mgo.Collection
books *mgo.Collection
tags *mgo.Collection
user *mgo.Collection
stats *mgo.Collection
}
func initDB() *DB {
var err error
d := new(DB)
d.session, err = mgo.Dial(DB_IP)
if err != nil {
panic(err)
}
database := d.session.DB(DB_NAME)
d.meta = database.C(META_COLL)
d.books = database.C(BOOKS_COLL)
d.tags = database.C(TAGS_COLL)
d.user = database.C(USERS_COLL)
d.stats = database.C(STATS_COLL)
return d
}
func (d *DB) Close() {
d.session.Close()
}
func md5Pass(pass string) []byte {
h := md5.New()
hash := h.Sum(([]byte)(PASS_SALT + pass))
return hash
}
func (d *DB) SetPassword(user string, pass string) error {
hash := md5Pass(pass)
return d.user.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}})
}
func (d *DB) UserValid(user string, pass string) bool {
hash := md5Pass(pass)
n, err := d.user.Find(bson.M{"user": user, "pass": hash}).Count()
if err != nil {
return false
}
return n != 0
}
func (d *DB) InsertStats(stats interface{}) error {
return d.stats.Insert(stats)
}
func (d *DB) InsertBook(book interface{}) error {
return d.books.Insert(book)
}
func (d *DB) RemoveBook(id bson.ObjectId) error {
return d.books.Remove(bson.M{"_id": id})
}
func (d *DB) UpdateBook(id bson.ObjectId, data interface{}) error {
return d.books.Update(bson.M{"_id": id}, bson.M{"$set": data})
}
func (d *DB) IncVisit(id bson.ObjectId) error {
return d.books.Update(bson.M{"_id": id}, bson.M{"$inc": bson.M{"VisitsCount": 1}})
}
func (d *DB) IncDownload(id bson.ObjectId) error {
return d.books.Update(bson.M{"_id": id}, bson.M{"$inc": bson.M{"DownloadCount": 1}})
}
/* optional parameters: length and start index
*
* Returns: list of books, number found and err
*/
func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) {
var start, length int
if len(r) > 0 {
length = r[0]
if len(r) > 1 {
start = r[1]
}
}
q := d.books.Find(query).Sort("-_id")
num, err = q.Count()
if err != nil {
return
}
if start != 0 {
q = q.Skip(start)
}
if length != 0 {
q = q.Limit(length)
}
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
/* Get the most visited books
*/
func (d *DB) GetVisitedBooks(num int) (books []Book, err error) {
var q *mgo.Query
q = d.books.Find(bson.M{"active": true}).Sort("-VisitsCount").Limit(num)
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
/* Get the most downloaded books
*/
func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) {
var q *mgo.Query
q = d.books.Find(bson.M{"active": true}).Sort("-DownloadCount").Limit(num)
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
/* optional parameters: length and start index
*
* Returns: list of books, number found and err
*/
func (d *DB) GetNewBooks(r ...int) (books []Book, num int, err error) {
return d.GetBooks(bson.M{"$nor": []bson.M{{"active": true}}}, r...)
}
func (d *DB) BookActive(id bson.ObjectId) bool {
var book Book
err := d.books.Find(bson.M{"_id": id}).One(&book)
if err != nil {
return false
}
return book.Active
}
func (d *DB) GetFS(prefix string) *mgo.GridFS {
return d.session.DB(DB_NAME).GridFS(prefix)
}
func (d *DB) areTagsOutdated() bool {
var result struct {
Id bson.ObjectId `bson:"_id"`
}
err := d.meta.Find(bson.M{"type": META_TYPE_TAGS}).One(&result)
if err != nil {
return true
}
lastUpdate := result.Id.Time()
return time.Since(lastUpdate).Minutes() > MINUTES_UPDATE_TAGS
}
func (d *DB) updateTags() error {
_, err := d.meta.RemoveAll(bson.M{"type": META_TYPE_TAGS})
if err != nil {
return err
}
var mr mgo.MapReduce
mr.Map = "function() { " +
"if (this.active) { this.subject.forEach(function(s) { emit(s, 1); }); }" +
"}"
mr.Reduce = "function(tag, vals) { " +
"var count = 0;" +
"vals.forEach(function() { count += 1; });" +
"return count;" +
"}"
mr.Out = bson.M{"replace": TAGS_COLL}
_, err = d.books.Find(bson.M{"active": true}).MapReduce(&mr, nil)
if err != nil {
return err
}
return d.meta.Insert(bson.M{"type": META_TYPE_TAGS})
}
func (d *DB) GetTags(numTags int) ([]string, error) {
if d.areTagsOutdated() {
err := d.updateTags()
if err != nil {
return nil, err
}
}
var result []struct {
Tag string "_id"
}
err := d.tags.Find(nil).Sort("-value").Limit(numTags).All(&result)
if err != nil {
return nil, err
}
tags := make([]string, len(result))
for i, r := range result {
tags[i] = r.Tag
}
return tags, nil
}

65
tools/getISBNnDesc/get.go
View file

@ -1,65 +0,0 @@
package main
import (
"fmt"
"git.gitorious.org/go-pkg/epubgo.git"
"labix.org/v2/mgo/bson"
)
func main() {
db = initDB()
defer db.Close()
books, _, _ := db.GetBooks(bson.M{})
for _, book := range books {
fmt.Println(book.Title)
e, err := OpenBook(book.File)
if err != nil {
fmt.Println("================", err)
continue
}
updateISBN(e, book)
updateDescription(e, book)
e.Close()
}
}
func updateISBN(e *epubgo.Epub, book Book) {
attr, err := e.MetadataAttr("identifier")
if err != nil {
fmt.Println("isbn ================", err)
return
}
data, err := e.Metadata("identifier")
if err != nil {
fmt.Println("isbn ================", err)
return
}
var isbn string
for i, d := range data {
if attr[i]["scheme"] == "ISBN" {
isbn = d
}
}
if isbn != "" {
db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"isbn": isbn})
}
}
func updateDescription(e *epubgo.Epub, book Book) {
descList, err := e.Metadata("description")
if err != nil {
fmt.Println("desc ================", err)
return
}
description := parseDescription(descList)
if len(description) < 10 {
return
}
if len(book.Description) < 10 || book.Description[:10] == description[:10] {
db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"description": description})
}
}

128
tools/getISBNnDesc/store.go
View file

@ -1,128 +0,0 @@
package main
import (
"bytes"
"git.gitorious.org/go-pkg/epubgo.git"
"io"
"io/ioutil"
"labix.org/v2/mgo/bson"
"regexp"
"strings"
)
func OpenBook(id bson.ObjectId) (*epubgo.Epub, error) {
fs := db.GetFS(FS_BOOKS)
f, err := fs.OpenId(id)
if err != nil {
return nil, err
}
defer f.Close()
buff, err := ioutil.ReadAll(f)
reader := bytes.NewReader(buff)
return epubgo.Load(reader, int64(len(buff)))
}
func StoreNewFile(name string, file io.Reader) (bson.ObjectId, error) {
fs := db.GetFS(FS_BOOKS)
fw, err := fs.Create(name)
if err != nil {
return "", err
}
defer fw.Close()
_, err = io.Copy(fw, file)
id, _ := fw.Id().(bson.ObjectId)
return id, err
}
func DeleteFile(id bson.ObjectId) error {
fs := db.GetFS(FS_BOOKS)
return fs.RemoveId(id)
}
func DeleteCover(id bson.ObjectId) error {
fs := db.GetFS(FS_IMGS)
return fs.RemoveId(id)
}
func DeleteBook(book Book) {
if book.Cover != "" {
DeleteCover(book.Cover)
}
if book.CoverSmall != "" {
DeleteCover(book.CoverSmall)
}
DeleteFile(book.File)
}
func cleanStr(str string) string {
str = strings.Replace(str, "&#39;", "'", -1)
exp, _ := regexp.Compile("&[^;]*;")
str = exp.ReplaceAllString(str, "")
exp, _ = regexp.Compile("[ ,]*$")
str = exp.ReplaceAllString(str, "")
return str
}
func parseAuthr(creator []string) []string {
exp1, _ := regexp.Compile("^(.*\\( *([^\\)]*) *\\))*$")
exp2, _ := regexp.Compile("^[^:]*: *(.*)$")
res := make([]string, len(creator))
for i, s := range creator {
auth := exp1.FindStringSubmatch(s)
if auth != nil {
res[i] = cleanStr(strings.Join(auth[2:], ", "))
} else {
auth := exp2.FindStringSubmatch(s)
if auth != nil {
res[i] = cleanStr(auth[1])
} else {
res[i] = cleanStr(s)
}
}
}
return res
}
func parseDescription(description []string) string {
str := cleanStr(strings.Join(description, "\n"))
str = strings.Replace(str, "</p>", "\n", -1)
exp, _ := regexp.Compile("<[^>]*>")
str = exp.ReplaceAllString(str, "")
str = strings.Replace(str, "&amp;", "&", -1)
str = strings.Replace(str, "&lt;", "<", -1)
str = strings.Replace(str, "&gt;", ">", -1)
str = strings.Replace(str, "\\n", "\n", -1)
return str
}
func parseSubject(subject []string) []string {
var res []string
for _, s := range subject {
res = append(res, strings.Split(s, " / ")...)
}
return res
}
func parseDate(date []string) string {
if len(date) == 0 {
return ""
}
return strings.Replace(date[0], "Unspecified: ", "", -1)
}
func keywords(b map[string]interface{}) (k []string) {
title, _ := b["title"].(string)
k = strings.Split(title, " ")
author, _ := b["author"].([]string)
for _, a := range author {
k = append(k, strings.Split(a, " ")...)
}
publisher, _ := b["publisher"].(string)
k = append(k, strings.Split(publisher, " ")...)
subject, _ := b["subject"].([]string)
k = append(k, subject...)
return
}

53
tools/importer/config.go
View file

@ -1,53 +0,0 @@
package main
const (
PORT = "8080"
DB_IP = "127.0.0.1"
DB_NAME = "trantor"
META_COLL = "meta"
BOOKS_COLL = "books"
TAGS_COLL = "tags"
VISITED_COLL = "visited"
DOWNLOADED_COLL = "downloaded"
HOURLY_VISITS_COLL = "visits.hourly"
DAILY_VISITS_COLL = "visits.daily"
MONTHLY_VISITS_COLL = "visits.monthly"
HOURLY_DOWNLOADS_COLL = "downloads.hourly"
DAILY_DOWNLOADS_COLL = "downloads.daily"
MONTHLY_DOWNLOADS_COLL = "downloads.monthly"
USERS_COLL = "users"
NEWS_COLL = "news"
STATS_COLL = "statistics"
FS_BOOKS = "fs_books"
FS_IMGS = "fs_imgs"
PASS_SALT = "ImperialLibSalt"
MINUTES_UPDATE_TAGS = 11
MINUTES_UPDATE_VISITED = 41
MINUTES_UPDATE_DOWNLOADED = 47
MINUTES_UPDATE_HOURLY_V = 31
MINUTES_UPDATE_DAILY_V = 60*12 + 7
MINUTES_UPDATE_MONTHLY_V = 60*24 + 11
MINUTES_UPDATE_HOURLY_D = 29
MINUTES_UPDATE_DAILY_D = 60*12 + 13
MINUTES_UPDATE_MONTHLY_D = 60*24 + 17
MINUTES_UPDATE_LOGGER = 5
TAGS_DISPLAY = 50
SEARCH_ITEMS_PAGE = 20
NEW_ITEMS_PAGE = 50
NUM_NEWS = 10
DAYS_NEWS_INDEXPAGE = 15
TEMPLATE_PATH = "templates/"
CSS_PATH = "css/"
JS_PATH = "js/"
IMG_PATH = "img/"
LOGGER_CONFIG = "logger.xml"
IMG_WIDTH_BIG = 300
IMG_WIDTH_SMALL = 60
IMG_QUALITY = 80
CHAN_SIZE = 100
)

202
tools/importer/cover.go
View file

@ -1,202 +0,0 @@
package main
import log "github.com/cihub/seelog"
import _ "image/png"
import _ "image/jpeg"
import _ "image/gif"
import (
"bytes"
"git.gitorious.org/go-pkg/epubgo.git"
"github.com/gorilla/mux"
"github.com/nfnt/resize"
"image"
"image/jpeg"
"io"
"io/ioutil"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"regexp"
"strings"
)
func coverHandler(h handler) {
vars := mux.Vars(h.r)
if !bson.IsObjectIdHex(vars["id"]) {
notFound(h)
return
}
id := bson.ObjectIdHex(vars["id"])
books, _, err := h.db.GetBooks(bson.M{"_id": id})
if err != nil || len(books) == 0 {
notFound(h)
return
}
book := books[0]
if !book.Active {
if !h.sess.IsAdmin() {
notFound(h)
return
}
}
fs := h.db.GetFS(FS_IMGS)
var f *mgo.GridFile
if vars["size"] == "small" {
f, err = fs.OpenId(book.CoverSmall)
} else {
f, err = fs.OpenId(book.Cover)
}
if err != nil {
log.Error("Error while opening image: ", err)
notFound(h)
return
}
defer f.Close()
headers := h.w.Header()
headers["Content-Type"] = []string{"image/jpeg"}
io.Copy(h.w, f)
}
func GetCover(e *epubgo.Epub, title string, db *DB) (bson.ObjectId, bson.ObjectId) {
imgId, smallId := coverFromMetadata(e, title, db)
if imgId != "" {
return imgId, smallId
}
imgId, smallId = searchCommonCoverNames(e, title, db)
if imgId != "" {
return imgId, smallId
}
/* search for img on the text */
exp, _ := regexp.Compile("<.*ima?g.*[(src)(href)]=[\"']([^\"']*(\\.[^\\.\"']*))[\"']")
it, errNext := e.Spine()
for errNext == nil {
file, err := it.Open()
if err != nil {
break
}
defer file.Close()
txt, err := ioutil.ReadAll(file)
if err != nil {
break
}
res := exp.FindSubmatch(txt)
if res != nil {
href := string(res[1])
urlPart := strings.Split(it.URL(), "/")
url := strings.Join(urlPart[:len(urlPart)-1], "/")
if href[:3] == "../" {
href = href[3:]
url = strings.Join(urlPart[:len(urlPart)-2], "/")
}
href = strings.Replace(href, "%20", " ", -1)
href = strings.Replace(href, "%27", "'", -1)
href = strings.Replace(href, "%28", "(", -1)
href = strings.Replace(href, "%29", ")", -1)
if url == "" {
url = href
} else {
url = url + "/" + href
}
img, err := e.OpenFile(url)
if err == nil {
defer img.Close()
return storeImg(img, title, db)
}
}
errNext = it.Next()
}
return "", ""
}
func coverFromMetadata(e *epubgo.Epub, title string, db *DB) (bson.ObjectId, bson.ObjectId) {
metaList, _ := e.MetadataAttr("meta")
for _, meta := range metaList {
if meta["name"] == "cover" {
img, err := e.OpenFileId(meta["content"])
if err == nil {
defer img.Close()
return storeImg(img, title, db)
}
}
}
return "", ""
}
func searchCommonCoverNames(e *epubgo.Epub, title string, db *DB) (bson.ObjectId, bson.ObjectId) {
for _, p := range []string{"cover.jpg", "Images/cover.jpg", "images/cover.jpg", "cover.jpeg", "cover1.jpg", "cover1.jpeg"} {
img, err := e.OpenFile(p)
if err == nil {
defer img.Close()
return storeImg(img, title, db)
}
}
return "", ""
}
func storeImg(img io.Reader, title string, db *DB) (bson.ObjectId, bson.ObjectId) {
/* open the files */
fBig, err := createCoverFile(title, db)
if err != nil {
log.Error("Error creating ", title, ": ", err.Error())
return "", ""
}
defer fBig.Close()
fSmall, err := createCoverFile(title+"_small", db)
if err != nil {
log.Error("Error creating ", title+"_small", ": ", err.Error())
return "", ""
}
defer fSmall.Close()
/* resize img */
var img2 bytes.Buffer
img1 := io.TeeReader(img, &img2)
jpgOptions := jpeg.Options{IMG_QUALITY}
imgResized, err := resizeImg(img1, IMG_WIDTH_BIG)
if err != nil {
log.Error("Error resizing big image: ", err.Error())
return "", ""
}
err = jpeg.Encode(fBig, imgResized, &jpgOptions)
if err != nil {
log.Error("Error encoding big image: ", err.Error())
return "", ""
}
imgSmallResized, err := resizeImg(&img2, IMG_WIDTH_SMALL)
if err != nil {
log.Error("Error resizing small image: ", err.Error())
return "", ""
}
err = jpeg.Encode(fSmall, imgSmallResized, &jpgOptions)
if err != nil {
log.Error("Error encoding small image: ", err.Error())
return "", ""
}
idBig, _ := fBig.Id().(bson.ObjectId)
idSmall, _ := fSmall.Id().(bson.ObjectId)
return idBig, idSmall
}
func createCoverFile(title string, db *DB) (*mgo.GridFile, error) {
fs := db.GetFS(FS_IMGS)
return fs.Create(title + ".jpg")
}
func resizeImg(imgReader io.Reader, width uint) (image.Image, error) {
img, _, err := image.Decode(imgReader)
if err != nil {
return nil, err
}
return resize.Resize(width, 0, img, resize.NearestNeighbor), nil
}

327
tools/importer/database.go
View file

@ -1,327 +0,0 @@
package main
import log "github.com/cihub/seelog"
import (
"crypto/md5"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"os"
"time"
)
type Book struct {
Id string `bson:"_id"`
Title string
Author []string
Contributor string
Publisher string
Description string
Subject []string
Date string
Lang []string
Isbn string
Type string
Format string
Source string
Relation string
Coverage string
Rights string
Meta string
File bson.ObjectId
FileSize int
Cover bson.ObjectId
CoverSmall bson.ObjectId
Active bool
Keywords []string
}
type News struct {
Date time.Time
Text string
}
type DB struct {
session *mgo.Session
}
func initDB() *DB {
var err error
d := new(DB)
d.session, err = mgo.Dial(DB_IP)
if err != nil {
log.Critical(err)
os.Exit(1)
}
return d
}
func (d *DB) Close() {
d.session.Close()
}
func (d *DB) Copy() *DB {
dbCopy := new(DB)
dbCopy.session = d.session.Copy()
return dbCopy
}
func md5Pass(pass string) []byte {
h := md5.New()
hash := h.Sum(([]byte)(PASS_SALT + pass))
return hash
}
func (d *DB) SetPassword(user string, pass string) error {
hash := md5Pass(pass)
userColl := d.session.DB(DB_NAME).C(USERS_COLL)
return userColl.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}})
}
func (d *DB) UserValid(user string, pass string) bool {
hash := md5Pass(pass)
userColl := d.session.DB(DB_NAME).C(USERS_COLL)
n, err := userColl.Find(bson.M{"user": user, "pass": hash}).Count()
if err != nil {
return false
}
return n != 0
}
func (d *DB) UserRole(user string) string {
type result struct {
Role string
}
res := result{}
userColl := d.session.DB(DB_NAME).C(USERS_COLL)
err := userColl.Find(bson.M{"user": user}).One(&res)
if err != nil {
return ""
}
return res.Role
}
func (d *DB) AddNews(text string) error {
var news News
news.Text = text
news.Date = time.Now()
newsColl := d.session.DB(DB_NAME).C(NEWS_COLL)
return newsColl.Insert(news)
}
func (d *DB) GetNews(num int, days int) (news []News, err error) {
query := bson.M{}
if days != 0 {
duration := time.Duration(-24*days) * time.Hour
date := time.Now().Add(duration)
query = bson.M{"date": bson.M{"$gt": date}}
}
newsColl := d.session.DB(DB_NAME).C(NEWS_COLL)
q := newsColl.Find(query).Sort("-date").Limit(num)
err = q.All(&news)
return
}
func (d *DB) InsertStats(stats interface{}) error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL)
return statsColl.Insert(stats)
}
func (d *DB) InsertBook(book interface{}) error {
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
return booksColl.Insert(book)
}
func (d *DB) RemoveBook(id bson.ObjectId) error {
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
return booksColl.Remove(bson.M{"_id": id})
}
func (d *DB) UpdateBook(id bson.ObjectId, data interface{}) error {
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
return booksColl.Update(bson.M{"_id": id}, bson.M{"$set": data})
}
/* optional parameters: length and start index
*
* Returns: list of books, number found and err
*/
func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) {
var start, length int
if len(r) > 0 {
length = r[0]
if len(r) > 1 {
start = r[1]
}
}
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
q := booksColl.Find(query).Sort("-_id")
num, err = q.Count()
if err != nil {
return
}
if start != 0 {
q = q.Skip(start)
}
if length != 0 {
q = q.Limit(length)
}
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
/* Get the most visited books
*/
func (d *DB) GetVisitedBooks(num int) (books []Book, err error) {
visitedColl := d.session.DB(DB_NAME).C(VISITED_COLL)
bookId, err := GetBooksVisited(num, visitedColl)
if err != nil {
return nil, err
}
books = make([]Book, num)
for i, id := range bookId {
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
booksColl.Find(bson.M{"_id": id}).One(&books[i])
books[i].Id = bson.ObjectId(books[i].Id).Hex()
}
return
}
func (d *DB) UpdateMostVisited() error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL)
mr := NewMR(d.session.DB(DB_NAME))
return mr.UpdateMostVisited(statsColl)
}
/* Get the most downloaded books
*/
func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) {
downloadedColl := d.session.DB(DB_NAME).C(DOWNLOADED_COLL)
bookId, err := GetBooksVisited(num, downloadedColl)
if err != nil {
return nil, err
}
books = make([]Book, num)
for i, id := range bookId {
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
booksColl.Find(bson.M{"_id": id}).One(&books[i])
books[i].Id = bson.ObjectId(books[i].Id).Hex()
}
return
}
func (d *DB) UpdateDownloadedBooks() error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL)
mr := NewMR(d.session.DB(DB_NAME))
return mr.UpdateMostDownloaded(statsColl)
}
/* optional parameters: length and start index
*
* Returns: list of books, number found and err
*/
func (d *DB) GetNewBooks(r ...int) (books []Book, num int, err error) {
return d.GetBooks(bson.M{"$nor": []bson.M{{"active": true}}}, r...)
}
func (d *DB) BookActive(id bson.ObjectId) bool {
var book Book
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
err := booksColl.Find(bson.M{"_id": id}).One(&book)
if err != nil {
return false
}
return book.Active
}
func (d *DB) GetFS(prefix string) *mgo.GridFS {
return d.session.DB(DB_NAME).GridFS(prefix)
}
func (d *DB) GetTags(numTags int) ([]string, error) {
tagsColl := d.session.DB(DB_NAME).C(TAGS_COLL)
return GetTags(numTags, tagsColl)
}
func (d *DB) UpdateTags() error {
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
mr := NewMR(d.session.DB(DB_NAME))
return mr.UpdateTags(booksColl)
}
type Visits struct {
Date int64 "_id"
Count int "value"
}
func (d *DB) GetHourVisits() ([]Visits, error) {
hourlyColl := d.session.DB(DB_NAME).C(HOURLY_VISITS_COLL)
return GetVisits(hourlyColl)
}
func (d *DB) UpdateHourVisits() error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL)
mr := NewMR(d.session.DB(DB_NAME))
return mr.UpdateHourVisits(statsColl)
}
func (d *DB) GetDayVisits() ([]Visits, error) {
dailyColl := d.session.DB(DB_NAME).C(DAILY_VISITS_COLL)
return GetVisits(dailyColl)
}
func (d *DB) UpdateDayVisits() error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL)
mr := NewMR(d.session.DB(DB_NAME))
return mr.UpdateDayVisits(statsColl)
}
func (d *DB) GetMonthVisits() ([]Visits, error) {
monthlyColl := d.session.DB(DB_NAME).C(MONTHLY_VISITS_COLL)
return GetVisits(monthlyColl)
}
func (d *DB) UpdateMonthVisits() error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL)
mr := NewMR(d.session.DB(DB_NAME))
return mr.UpdateMonthVisits(statsColl)
}
func (d *DB) GetHourDownloads() ([]Visits, error) {
hourlyColl := d.session.DB(DB_NAME).C(HOURLY_DOWNLOADS_COLL)
return GetVisits(hourlyColl)
}
func (d *DB) UpdateHourDownloads() error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL)
mr := NewMR(d.session.DB(DB_NAME))
return mr.UpdateHourDownloads(statsColl)
}
func (d *DB) GetDayDownloads() ([]Visits, error) {
dailyColl := d.session.DB(DB_NAME).C(DAILY_DOWNLOADS_COLL)
return GetVisits(dailyColl)
}
func (d *DB) UpdateDayDownloads() error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL)
mr := NewMR(d.session.DB(DB_NAME))
return mr.UpdateDayDownloads(statsColl)
}
func (d *DB) GetMonthDownloads() ([]Visits, error) {
monthlyColl := d.session.DB(DB_NAME).C(MONTHLY_DOWNLOADS_COLL)
return GetVisits(monthlyColl)
}
func (d *DB) UpdateMonthDownloads() error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL)
mr := NewMR(d.session.DB(DB_NAME))
return mr.UpdateMonthDownloads(statsColl)
}

65
tools/importer/importer.go
View file

@ -1,65 +0,0 @@
package main
import log "github.com/cihub/seelog"
import (
"git.gitorious.org/go-pkg/epubgo.git"
"net/http"
"os"
)
func main() {
db := initDB()
defer db.Close()
for _, file := range os.Args[1:len(os.Args)] {
uploadEpub(file, db)
}
}
func uploadEpub(filename string, db *DB) {
epub, err := epubgo.Open(filename)
if err != nil {
log.Error("Not valid epub '", filename, "': ", err)
return
}
defer epub.Close()
book := parseFile(epub, db)
title, _ := book["title"].(string)
_, numTitleFound, _ := db.GetBooks(buildQuery("title:"+title), 1)
if numTitleFound == 0 {
book["active"] = true
}
file, _ := os.Open(filename)
defer file.Close()
id, size, err := StoreNewFile(title+".epub", file, db)
if err != nil {
log.Error("Error storing book (", title, "): ", err)
return
}
book["file"] = id
book["filesize"] = size
err = db.InsertBook(book)
if err != nil {
log.Error("Error storing metadata (", title, "): ", err)
return
}
log.Info("File uploaded: ", filename)
}
type Status struct {
Upload bool
Stats bool
Search string
}
func GetStatus(h handler) Status {
return Status{}
}
func loadTemplate(w http.ResponseWriter, tmpl string, data interface{}) {}
func loadTxtTemplate(w http.ResponseWriter, tmpl string, data interface{}) {}
func notFound(h handler) {}

283
tools/importer/mapreduce.go
View file

@ -1,283 +0,0 @@
package main
import (
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"time"
)
func GetTags(numTags int, tagsColl *mgo.Collection) ([]string, error) {
var result []struct {
Tag string "_id"
}
err := tagsColl.Find(nil).Sort("-value").Limit(numTags).All(&result)
if err != nil {
return nil, err
}
tags := make([]string, len(result))
for i, r := range result {
tags[i] = r.Tag
}
return tags, nil
}
func GetBooksVisited(num int, visitedColl *mgo.Collection) ([]bson.ObjectId, error) {
var result []struct {
Book bson.ObjectId "_id"
}
err := visitedColl.Find(nil).Sort("-value").Limit(num).All(&result)
if err != nil {
return nil, err
}
books := make([]bson.ObjectId, len(result))
for i, r := range result {
books[i] = r.Book
}
return books, nil
}
func GetVisits(visitsColl *mgo.Collection) ([]Visits, error) {
var result []Visits
err := visitsColl.Find(nil).All(&result)
return result, err
}
type MR struct {
database *mgo.Database
}
func NewMR(database *mgo.Database) *MR {
m := new(MR)
m.database = database
return m
}
func (m *MR) UpdateTags(booksColl *mgo.Collection) error {
var mr mgo.MapReduce
mr.Map = `function() {
if (this.subject) {
this.subject.forEach(function(s) { emit(s, 1); });
}
}`
mr.Reduce = `function(tag, vals) {
var count = 0;
vals.forEach(function() { count += 1; });
return count;
}`
return m.update(&mr, bson.M{"active": true}, booksColl, TAGS_COLL)
}
func (m *MR) UpdateMostVisited(statsColl *mgo.Collection) error {
var mr mgo.MapReduce
mr.Map = `function() {
if (this.id) {
emit(this.id, 1);
}
}`
mr.Reduce = `function(tag, vals) {
var count = 0;
vals.forEach(function() { count += 1; });
return count;
}`
return m.update(&mr, bson.M{"section": "book"}, statsColl, VISITED_COLL)
}
func (m *MR) UpdateMostDownloaded(statsColl *mgo.Collection) error {
var mr mgo.MapReduce
mr.Map = `function() {
emit(this.id, 1);
}`
mr.Reduce = `function(tag, vals) {
var count = 0;
vals.forEach(function() { count += 1; });
return count;
}`
return m.update(&mr, bson.M{"section": "download"}, statsColl, DOWNLOADED_COLL)
}
func (m *MR) UpdateHourVisits(statsColl *mgo.Collection) error {
const numDays = 2
start := time.Now().UTC().Add(-numDays * 24 * time.Hour)
const reduce = `function(date, vals) {
var count = 0;
vals.forEach(function(v) { count += v; });
return count;
}`
var mr mgo.MapReduce
mr.Map = `function() {
var date = Date.UTC(this.date.getUTCFullYear(),
this.date.getUTCMonth(),
this.date.getUTCDate(),
this.date.getUTCHours());
emit({date: date, session: this.session}, 1);
}`
mr.Reduce = reduce
err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, HOURLY_VISITS_COLL+"_raw")
if err != nil {
return err
}
var mr2 mgo.MapReduce
mr2.Map = `function() {
emit(this['_id']['date'], 1);
}`
mr2.Reduce = reduce
hourly_raw := m.database.C(HOURLY_VISITS_COLL + "_raw")
return m.update(&mr2, bson.M{}, hourly_raw, HOURLY_VISITS_COLL)
}
func (m *MR) UpdateDayVisits(statsColl *mgo.Collection) error {
const numDays = 30
start := time.Now().UTC().Add(-numDays * 24 * time.Hour).Truncate(24 * time.Hour)
const reduce = `function(date, vals) {
var count = 0;
vals.forEach(function(v) { count += v; });
return count;
}`
var mr mgo.MapReduce
mr.Map = `function() {
var date = Date.UTC(this.date.getUTCFullYear(),
this.date.getUTCMonth(),
this.date.getUTCDate());
emit({date: date, session: this.session}, 1);
}`
mr.Reduce = reduce
err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, DAILY_VISITS_COLL+"_raw")
if err != nil {
return err
}
var mr2 mgo.MapReduce
mr2.Map = `function() {
emit(this['_id']['date'], 1);
}`
mr2.Reduce = reduce
daily_raw := m.database.C(DAILY_VISITS_COLL + "_raw")
return m.update(&mr2, bson.M{}, daily_raw, DAILY_VISITS_COLL)
}
func (m *MR) UpdateMonthVisits(statsColl *mgo.Collection) error {
const numDays = 365
start := time.Now().UTC().Add(-numDays * 24 * time.Hour).Truncate(24 * time.Hour)
const reduce = `function(date, vals) {
var count = 0;
vals.forEach(function(v) { count += v; });
return count;
}`
var mr mgo.MapReduce
mr.Map = `function() {
var date = Date.UTC(this.date.getUTCFullYear(),
this.date.getUTCMonth());
emit({date: date, session: this.session}, 1);
}`
mr.Reduce = reduce
err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, MONTHLY_VISITS_COLL+"_raw")
if err != nil {
return err
}
var mr2 mgo.MapReduce
mr2.Map = `function() {
emit(this['_id']['date'], 1);
}`
mr2.Reduce = reduce
monthly_raw := m.database.C(MONTHLY_VISITS_COLL + "_raw")
return m.update(&mr2, bson.M{}, monthly_raw, MONTHLY_VISITS_COLL)
}
func (m *MR) UpdateHourDownloads(statsColl *mgo.Collection) error {
const numDays = 2
start := time.Now().UTC().Add(-numDays * 24 * time.Hour)
var mr mgo.MapReduce
mr.Map = `function() {
if (this.section == "download") {
var date = Date.UTC(this.date.getUTCFullYear(),
this.date.getUTCMonth(),
this.date.getUTCDate(),
this.date.getUTCHours());
emit(date, 1);
}
}`
mr.Reduce = `function(date, vals) {
var count = 0;
vals.forEach(function(v) { count += v; });
return count;
}`
return m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, HOURLY_DOWNLOADS_COLL)
}
func (m *MR) UpdateDayDownloads(statsColl *mgo.Collection) error {
const numDays = 30
start := time.Now().UTC().Add(-numDays * 24 * time.Hour).Truncate(24 * time.Hour)
var mr mgo.MapReduce
mr.Map = `function() {
if (this.section == "download") {
var date = Date.UTC(this.date.getUTCFullYear(),
this.date.getUTCMonth(),
this.date.getUTCDate());
emit(date, 1);
}
}`
mr.Reduce = `function(date, vals) {
var count = 0;
vals.forEach(function(v) { count += v; });
return count;
}`
return m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, DAILY_DOWNLOADS_COLL)
}
func (m *MR) UpdateMonthDownloads(statsColl *mgo.Collection) error {
const numDays = 365
start := time.Now().UTC().Add(-numDays * 24 * time.Hour).Truncate(24 * time.Hour)
var mr mgo.MapReduce
mr.Map = `function() {
if (this.section == "download") {
var date = Date.UTC(this.date.getUTCFullYear(),
this.date.getUTCMonth());
emit(date, 1);
}
}`
mr.Reduce = `function(date, vals) {
var count = 0;
vals.forEach(function(v) { count += v; });
return count;
}`
return m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, MONTHLY_DOWNLOADS_COLL)
}
func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection, storeColl string) error {
metaColl := m.database.C(META_COLL)
_, err := metaColl.RemoveAll(bson.M{"type": storeColl})
if err != nil {
return err
}
mr.Out = bson.M{"replace": storeColl}
_, err = queryColl.Find(query).MapReduce(mr, nil)
if err != nil {
return err
}
return metaColl.Insert(bson.M{"type": storeColl})
}
func (m *MR) isOutdated(coll string, minutes float64) bool {
var result struct {
Id bson.ObjectId `bson:"_id"`
}
metaColl := m.database.C(META_COLL)
err := metaColl.Find(bson.M{"type": coll}).One(&result)
if err != nil {
return true
}
lastUpdate := result.Id.Time()
return time.Since(lastUpdate).Minutes() > minutes
}

85
tools/importer/search.go
View file

@ -1,85 +0,0 @@
package main
import (
"labix.org/v2/mgo/bson"
"net/http"
"strconv"
"strings"
)
func buildQuery(q string) bson.M {
var reg []bson.RegEx
query := bson.M{"active": true}
words := strings.Split(q, " ")
for _, w := range words {
tag := strings.SplitN(w, ":", 2)
if len(tag) > 1 {
query[tag[0]] = bson.RegEx{tag[1], "i"}
} else {
reg = append(reg, bson.RegEx{w, "i"})
}
}
if len(reg) > 0 {
query["keywords"] = bson.M{"$all": reg}
}
return query
}
type searchData struct {
S Status
Found int
Books []Book
ItemsPage int
Page int
Next string
Prev string
}
func searchHandler(h handler) {
err := h.r.ParseForm()
if err != nil {
http.Error(h.w, err.Error(), http.StatusInternalServerError)
return
}
req := strings.Join(h.r.Form["q"], " ")
page := 0
if len(h.r.Form["p"]) != 0 {
page, err = strconv.Atoi(h.r.Form["p"][0])
if err != nil {
page = 0
}
}
items_page := itemsPage(h.r)
res, num, _ := h.db.GetBooks(buildQuery(req), items_page, page*items_page)
var data searchData
data.S = GetStatus(h)
data.S.Search = req
data.Books = res
data.ItemsPage = items_page
data.Found = num
data.Page = page + 1
if num > (page+1)*items_page {
data.Next = "/search/?q=" + req + "&p=" + strconv.Itoa(page+1) + "&num=" + strconv.Itoa(items_page)
}
if page > 0 {
data.Prev = "/search/?q=" + req + "&p=" + strconv.Itoa(page-1) + "&num=" + strconv.Itoa(items_page)
}
format := h.r.Form["fmt"]
if (len(format) > 0) && (format[0] == "rss") {
loadTxtTemplate(h.w, "search_rss.xml", data)
} else {
loadTemplate(h.w, "search", data)
}
}
func itemsPage(r *http.Request) int {
if len(r.Form["num"]) > 0 {
items_page, err := strconv.Atoi(r.Form["num"][0])
if err == nil {
return items_page
}
}
return SEARCH_ITEMS_PAGE
}

81
tools/importer/session.go
View file

@ -1,81 +0,0 @@
package main
import (
"encoding/hex"
"github.com/gorilla/securecookie"
"github.com/gorilla/sessions"
"net/http"
)
var sesStore = sessions.NewCookieStore(securecookie.GenerateRandomKey(64))
type Notification struct {
Title string
Msg string
Type string /* error, info or success */
}
type Session struct {
User string
Role string
S *sessions.Session
}
func GetSession(r *http.Request, db *DB) (s *Session) {
s = new(Session)
var err error
s.S, err = sesStore.Get(r, "session")
if err == nil && !s.S.IsNew {
s.User, _ = s.S.Values["user"].(string)
s.Role = db.UserRole(s.User)
}
if s.S.IsNew {
s.S.Values["id"] = hex.EncodeToString(securecookie.GenerateRandomKey(16))
}
return
}
func (s *Session) GetNotif() []Notification {
session := s.S
msgs := session.Flashes("nMsg")
titles := session.Flashes("nTitle")
tpes := session.Flashes("nType")
notif := make([]Notification, len(msgs))
for i, m := range msgs {
msg, _ := m.(string)
title, _ := titles[i].(string)
tpe, _ := tpes[i].(string)
notif[i] = Notification{title, msg, tpe}
}
return notif
}
func (s *Session) LogIn(user string) {
s.User = user
s.S.Values["user"] = user
}
func (s *Session) LogOut() {
s.S.Values["user"] = ""
}
func (s *Session) Notify(title, msg, tpe string) {
s.S.AddFlash(msg, "nMsg")
s.S.AddFlash(title, "nTitle")
s.S.AddFlash(tpe, "nType")
}
func (s *Session) Save(w http.ResponseWriter, r *http.Request) {
sesStore.Save(r, w, s.S)
}
func (s *Session) Id() string {
id, _ := s.S.Values["id"].(string)
return id
}
func (s *Session) IsAdmin() bool {
return s.Role == "admin"
}

244
tools/importer/stats.go
View file

@ -1,244 +0,0 @@
package main
import log "github.com/cihub/seelog"
import (
"github.com/gorilla/mux"
"labix.org/v2/mgo/bson"
"net/http"
"strconv"
"strings"
"time"
)
type handler struct {
w http.ResponseWriter
r *http.Request
sess *Session
db *DB
}
func InitStats(database *DB) {
statsChannel = make(chan statsRequest, CHAN_SIZE)
go statsWorker(database)
}
func GatherStats(function func(handler), database *DB) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
log.Info("Query ", r.Method, " ", r.RequestURI)
var h handler
h.db = database.Copy()
defer h.db.Close()
h.w = w
h.r = r
h.sess = GetSession(r, h.db)
function(h)
statsChannel <- statsRequest{bson.Now(), mux.Vars(r), h.sess, r}
}
}
var statsChannel chan statsRequest
type statsRequest struct {
date time.Time
vars map[string]string
sess *Session
r *http.Request
}
func statsWorker(database *DB) {
db := database.Copy()
defer db.Close()
for req := range statsChannel {
stats := make(map[string]interface{})
appendFiles(req.r, stats)
appendMuxVars(req.vars, stats)
appendUrl(req.r, stats)
appendSession(req.sess, stats)
stats["method"] = req.r.Method
stats["date"] = req.date
db.InsertStats(stats)
}
}
func statsHandler(h handler) {
var data statsData
data.S = GetStatus(h)
data.S.Stats = true
data.HVisits = getHourlyVisits(h.db)
data.DVisits = getDailyVisits(h.db)
data.MVisits = getMonthlyVisits(h.db)
data.HDownloads = getHourlyDownloads(h.db)
data.DDownloads = getDailyDownloads(h.db)
data.MDownloads = getMonthlyDownloads(h.db)
loadTemplate(h.w, "stats", data)
}
type statsData struct {
S Status
HVisits []visitData
DVisits []visitData
MVisits []visitData
HDownloads []visitData
DDownloads []visitData
MDownloads []visitData
}
type visitData struct {
Label string
Count int
}
func getHourlyVisits(db *DB) []visitData {
var visits []visitData
visit, _ := db.GetHourVisits()
for _, v := range visit {
var elem visitData
hour := time.Unix(v.Date/1000, 0).UTC().Hour()
elem.Label = strconv.Itoa(hour + 1)
elem.Count = v.Count
visits = append(visits, elem)
}
return visits
}
func getDailyVisits(db *DB) []visitData {
var visits []visitData
visit, _ := db.GetDayVisits()
for _, v := range visit {
var elem visitData
day := time.Unix(v.Date/1000, 0).UTC().Day()
elem.Label = strconv.Itoa(day)
elem.Count = v.Count
visits = append(visits, elem)
}
return visits
}
func getMonthlyVisits(db *DB) []visitData {
var visits []visitData
visit, _ := db.GetMonthVisits()
for _, v := range visit {
var elem visitData
month := time.Unix(v.Date/1000, 0).UTC().Month()
elem.Label = month.String()
elem.Count = v.Count
visits = append(visits, elem)
}
return visits
}
func getHourlyDownloads(db *DB) []visitData {
var visits []visitData
visit, _ := db.GetHourDownloads()
for _, v := range visit {
var elem visitData
hour := time.Unix(v.Date/1000, 0).UTC().Hour()
elem.Label = strconv.Itoa(hour + 1)
elem.Count = v.Count
visits = append(visits, elem)
}
return visits
}
func getDailyDownloads(db *DB) []visitData {
var visits []visitData
visit, _ := db.GetDayDownloads()
for _, v := range visit {
var elem visitData
day := time.Unix(v.Date/1000, 0).UTC().Day()
elem.Label = strconv.Itoa(day)
elem.Count = v.Count
visits = append(visits, elem)
}
return visits
}
func getMonthlyDownloads(db *DB) []visitData {
var visits []visitData
visit, _ := db.GetMonthDownloads()
for _, v := range visit {
var elem visitData
month := time.Unix(v.Date/1000, 0).UTC().Month()
elem.Label = month.String()
elem.Count = v.Count
visits = append(visits, elem)
}
return visits
}
func appendFiles(r *http.Request, stats map[string]interface{}) {
if r.Method == "POST" && r.MultipartForm != nil {
files := r.MultipartForm.File
for key := range files {
list := make([]string, len(files[key]))
for i, f := range files[key] {
list[i] = f.Filename
}
stats[key] = list
}
}
}
func appendMuxVars(vars map[string]string, stats map[string]interface{}) {
for key, value := range vars {
switch {
case key == "id":
if bson.IsObjectIdHex(value) {
stats["id"] = bson.ObjectIdHex(value)
}
case key == "ids":
var objectIds []bson.ObjectId
ids := strings.Split(value, "/")
for _, id := range ids {
if bson.IsObjectIdHex(value) {
objectIds = append(objectIds, bson.ObjectIdHex(id))
}
}
if len(objectIds) > 0 {
stats["ids"] = objectIds
stats["id"] = objectIds[0]
}
default:
stats[key] = value
}
}
}
func appendUrl(r *http.Request, stats map[string]interface{}) {
for key, value := range r.URL.Query() {
stats[key] = value
}
stats["host"] = r.Host
stats["path"] = r.URL.Path
pattern := strings.Split(r.URL.Path, "/")
if len(pattern) > 1 && pattern[1] != "" {
stats["section"] = pattern[1]
} else {
stats["section"] = "/"
}
}
func appendSession(sess *Session, stats map[string]interface{}) {
stats["session"] = sess.Id()
if sess.User != "" {
stats["user"] = sess.User
}
}

128
tools/importer/store.go
View file

@ -1,128 +0,0 @@
package main
import (
"bytes"
"git.gitorious.org/go-pkg/epubgo.git"
"io"
"io/ioutil"
"labix.org/v2/mgo/bson"
"regexp"
"strings"
)
func OpenBook(id bson.ObjectId, db *DB) (*epubgo.Epub, error) {
fs := db.GetFS(FS_BOOKS)
f, err := fs.OpenId(id)
if err != nil {
return nil, err
}
defer f.Close()
buff, err := ioutil.ReadAll(f)
reader := bytes.NewReader(buff)
return epubgo.Load(reader, int64(len(buff)))
}
func StoreNewFile(name string, file io.Reader, db *DB) (bson.ObjectId, int64, error) {
fs := db.GetFS(FS_BOOKS)
fw, err := fs.Create(name)
if err != nil {
return "", 0, err
}
defer fw.Close()
size, err := io.Copy(fw, file)
id, _ := fw.Id().(bson.ObjectId)
return id, size, err
}
func DeleteFile(id bson.ObjectId, db *DB) error {
fs := db.GetFS(FS_BOOKS)
return fs.RemoveId(id)
}
func DeleteCover(id bson.ObjectId, db *DB) error {
fs := db.GetFS(FS_IMGS)
return fs.RemoveId(id)
}
func DeleteBook(book Book, db *DB) {
if book.Cover != "" {
DeleteCover(book.Cover, db)
}
if book.CoverSmall != "" {
DeleteCover(book.CoverSmall, db)
}
DeleteFile(book.File, db)
}
func cleanStr(str string) string {
str = strings.Replace(str, "&#39;", "'", -1)
exp, _ := regexp.Compile("&[^;]*;")
str = exp.ReplaceAllString(str, "")
exp, _ = regexp.Compile("[ ,]*$")
str = exp.ReplaceAllString(str, "")
return str
}
func parseAuthr(creator []string) []string {
exp1, _ := regexp.Compile("^(.*\\( *([^\\)]*) *\\))*$")
exp2, _ := regexp.Compile("^[^:]*: *(.*)$")
res := make([]string, len(creator))
for i, s := range creator {
auth := exp1.FindStringSubmatch(s)
if auth != nil {
res[i] = cleanStr(strings.Join(auth[2:], ", "))
} else {
auth := exp2.FindStringSubmatch(s)
if auth != nil {
res[i] = cleanStr(auth[1])
} else {
res[i] = cleanStr(s)
}
}
}
return res
}
func parseDescription(description []string) string {
str := cleanStr(strings.Join(description, "\n"))
str = strings.Replace(str, "</p>", "\n", -1)
exp, _ := regexp.Compile("<[^>]*>")
str = exp.ReplaceAllString(str, "")
str = strings.Replace(str, "&amp;", "&", -1)
str = strings.Replace(str, "&lt;", "<", -1)
str = strings.Replace(str, "&gt;", ">", -1)
str = strings.Replace(str, "\\n", "\n", -1)
return str
}
func parseSubject(subject []string) []string {
var res []string
for _, s := range subject {
res = append(res, strings.Split(s, " / ")...)
}
return res
}
func parseDate(date []string) string {
if len(date) == 0 {
return ""
}
return strings.Replace(date[0], "Unspecified: ", "", -1)
}
func keywords(b map[string]interface{}) (k []string) {
title, _ := b["title"].(string)
k = strings.Split(title, " ")
author, _ := b["author"].([]string)
for _, a := range author {
k = append(k, strings.Split(a, " ")...)
}
publisher, _ := b["publisher"].(string)
k = append(k, strings.Split(publisher, " ")...)
subject, _ := b["subject"].([]string)
k = append(k, subject...)
return
}

146
tools/importer/upload.go
View file

@ -1,146 +0,0 @@
package main
import log "github.com/cihub/seelog"
import (
"bytes"
"git.gitorious.org/go-pkg/epubgo.git"
"io/ioutil"
"mime/multipart"
"strings"
)
func InitUpload(database *DB) {
uploadChannel = make(chan uploadRequest, CHAN_SIZE)
go uploadWorker(database)
}
var uploadChannel chan uploadRequest
type uploadRequest struct {
file multipart.File
filename string
}
func uploadWorker(database *DB) {
db := database.Copy()
defer db.Close()
for req := range uploadChannel {
processFile(req, db)
}
}
func processFile(req uploadRequest, db *DB) {
defer req.file.Close()
epub, err := openMultipartEpub(req.file)
if err != nil {
log.Warn("Not valid epub uploaded file ", req.filename, ": ", err)
return
}
defer epub.Close()
book := parseFile(epub, db)
title, _ := book["title"].(string)
req.file.Seek(0, 0)
id, size, err := StoreNewFile(title+".epub", req.file, db)
if err != nil {
log.Error("Error storing book (", title, "): ", err)
return
}
book["file"] = id
book["filesize"] = size
err = db.InsertBook(book)
if err != nil {
log.Error("Error storing metadata (", title, "): ", err)
return
}
log.Info("File uploaded: ", req.filename)
}
func uploadPostHandler(h handler) {
problem := false
h.r.ParseMultipartForm(20000000)
filesForm := h.r.MultipartForm.File["epub"]
for _, f := range filesForm {
file, err := f.Open()
if err != nil {
log.Error("Can not open uploaded file ", f.Filename, ": ", err)
h.sess.Notify("Upload problem!", "There was a problem with book "+f.Filename, "error")
problem = true
continue
}
uploadChannel <- uploadRequest{file, f.Filename}
}
if !problem {
if len(filesForm) > 0 {
h.sess.Notify("Upload successful!", "Thank you for your contribution", "success")
} else {
h.sess.Notify("Upload problem!", "No books where uploaded.", "error")
}
}
uploadHandler(h)
}
func uploadHandler(h handler) {
var data uploadData
data.S = GetStatus(h)
data.S.Upload = true
loadTemplate(h.w, "upload", data)
}
type uploadData struct {
S Status
}
func openMultipartEpub(file multipart.File) (*epubgo.Epub, error) {
buff, _ := ioutil.ReadAll(file)
reader := bytes.NewReader(buff)
return epubgo.Load(reader, int64(len(buff)))
}
func parseFile(epub *epubgo.Epub, db *DB) map[string]interface{} {
book := map[string]interface{}{}
for _, m := range epub.MetadataFields() {
data, err := epub.Metadata(m)
if err != nil {
continue
}
switch m {
case "creator":
book["author"] = parseAuthr(data)
case "description":
book[m] = parseDescription(data)
case "subject":
book[m] = parseSubject(data)
case "date":
book[m] = parseDate(data)
case "language":
book["lang"] = data
case "title", "contributor", "publisher":
book[m] = cleanStr(strings.Join(data, ", "))
case "identifier":
attr, _ := epub.MetadataAttr(m)
for i, d := range data {
if attr[i]["scheme"] == "ISBN" {
book["isbn"] = d
}
}
default:
book[m] = strings.Join(data, ", ")
}
}
title, _ := book["title"].(string)
book["file"] = nil
cover, coverSmall := GetCover(epub, title, db)
if cover != "" {
book["cover"] = cover
book["coversmall"] = coverSmall
}
book["keywords"] = keywords(book)
return book
}

40
tools/keywords/config.go
View file

@ -1,40 +0,0 @@
package main
const (
PORT = "8080"
DB_IP = "127.0.0.1"
DB_NAME = "trantor"
META_COLL = "meta"
FS_BOOKS = "fs_books"
FS_IMGS = "fs_imgs"
PASS_SALT = "ImperialLibSalt"
MINUTES_UPDATE_TAGS = 11
MINUTES_UPDATE_VISITED = 41
MINUTES_UPDATE_DOWNLOADED = 47
MINUTES_UPDATE_HOURLY_V = 31
MINUTES_UPDATE_DAILY_V = 60*12 + 7
MINUTES_UPDATE_MONTHLY_V = 60*24 + 11
MINUTES_UPDATE_HOURLY_D = 29
MINUTES_UPDATE_DAILY_D = 60*12 + 13
MINUTES_UPDATE_MONTHLY_D = 60*24 + 17
MINUTES_UPDATE_LOGGER = 5
BOOKS_FRONT_PAGE = 6
SEARCH_ITEMS_PAGE = 20
NEW_ITEMS_PAGE = 50
NUM_NEWS = 10
DAYS_NEWS_INDEXPAGE = 15
TEMPLATE_PATH = "templates/"
CSS_PATH = "css/"
JS_PATH = "js/"
IMG_PATH = "img/"
LOGGER_CONFIG = "logger.xml"
IMG_WIDTH_BIG = 300
IMG_WIDTH_SMALL = 60
IMG_QUALITY = 80
CHAN_SIZE = 100
)

63
tools/keywords/keywords.go
View file

@ -1,63 +0,0 @@
package main
import (
"fmt"
"git.gitorious.org/trantor/trantor.git/database"
"gopkgs.com/unidecode.v1"
"labix.org/v2/mgo/bson"
"strings"
"unicode"
)
func main() {
db := database.Init(DB_IP, DB_NAME)
defer db.Close()
books, _, err := db.GetBooks(bson.M{}, 0, 0)
if err != nil {
fmt.Println(err)
return
}
for _, b := range books {
fmt.Println(b.Title)
book := map[string]interface{}{
"title": b.Title,
"author": b.Author,
"publisher": b.Publisher,
"subject": b.Subject,
}
k := keywords(book)
book = map[string]interface{}{"keywords": k}
id := bson.ObjectIdHex(b.Id)
err := db.UpdateBook(id, book)
if err != nil {
fmt.Println(err)
}
}
}
func keywords(b map[string]interface{}) (k []string) {
title, _ := b["title"].(string)
k = tokens(title)
author, _ := b["author"].([]string)
for _, a := range author {
k = append(k, tokens(a)...)
}
publisher, _ := b["publisher"].(string)
k = append(k, tokens(publisher)...)
subject, _ := b["subject"].([]string)
for _, s := range subject {
k = append(k, tokens(s)...)
}
return
}
func tokens(str string) []string {
str = unidecode.Unidecode(str)
str = strings.ToLower(str)
f := func(r rune) bool {
return unicode.IsControl(r) || unicode.IsPunct(r) || unicode.IsSpace(r)
}
return strings.FieldsFunc(str, f)
}

148
tools/store/store.go
View file

@ -1,148 +0,0 @@
package main
import (
log "github.com/cihub/seelog"
"crypto/rand"
"encoding/base64"
"os"
"git.gitorious.org/trantor/trantor.git/storage"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
const (
DB_IP = "127.0.0.1"
DB_NAME = "trantor"
BOOKS_COLL = "books"
FS_BOOKS = "fs_books"
FS_IMGS = "fs_imgs"
STORE_PATH = "store/"
EPUB_FILE = "book.epub"
COVER_FILE = "cover.jpg"
COVER_SMALL_FILE = "coverSmall.jpg"
NUM_WORKERS = 10
)
type Book struct {
Id bson.ObjectId `bson:"_id"`
Title string
Author []string
Contributor string
Publisher string
Description string
Subject []string
Date string
Lang []string
Isbn string
Type string
Format string
Source string
Relation string
Coverage string
Rights string
Meta string
File bson.ObjectId
FileSize int
Cover bson.ObjectId
CoverSmall bson.ObjectId
Active bool
Keywords []string
}
func main() {
db := InitDB(DB_IP)
defer db.Close()
store, err := storage.Init(STORE_PATH)
if err != nil {
log.Critical(err)
os.Exit(1)
}
channel := make(chan Book)
quit := make(chan bool)
for i := 0; i < NUM_WORKERS; i++ {
go worker(channel, quit, db, store)
}
booksColl := db.DB(DB_NAME).C(BOOKS_COLL)
books := booksColl.Find(bson.M{}).Batch(200).Prefetch(0.25).Iter()
var book Book
for books.Next(&book) {
channel <- book
}
if err := books.Close(); err != nil {
log.Critical(err)
}
close(channel)
for i := 0; i < NUM_WORKERS; i++ {
log.Info("Worker ", i, " has finished")
<-quit
}
}
func InitDB(host string) *mgo.Session {
session, err := mgo.Dial(host)
if err != nil {
log.Critical(err)
os.Exit(1)
}
return session
}
func worker(channel chan Book, quit chan bool, database *mgo.Session, store *storage.Store) {
db := database.Copy()
defer db.Close()
fsBooks := db.DB(DB_NAME).GridFS(FS_BOOKS)
fsImgs := db.DB(DB_NAME).GridFS(FS_IMGS)
booksColl := db.DB(DB_NAME).C(BOOKS_COLL)
for book := range channel {
id := genId()
log.Info("== Storing book '", book.Title, "' (", id, ") ==")
cover := true
process(id, EPUB_FILE, book.File, fsBooks, store)
err := process(id, COVER_FILE, book.Cover, fsImgs, store)
if err != nil {
cover = false
}
process(id, COVER_SMALL_FILE, book.CoverSmall, fsImgs, store)
query := bson.M{"$set": bson.M{"id": id, "cover": cover},
"$unset": bson.M{"file": "", "coversmall": ""}}
err = booksColl.UpdateId(book.Id, query)
if err != nil {
log.Error("Can no update ", book.Id.Hex())
}
}
quit <- true
}
func process(id string, name string, objId bson.ObjectId, fs *mgo.GridFS, store *storage.Store) error {
f, err := fs.OpenId(objId)
if err != nil {
if name == EPUB_FILE {
log.Error(id, " - can not open ", objId.Hex())
}
return err
}
defer f.Close()
_, err = store.Store(id, f, name)
if err != nil {
log.Error("Can not store '", id, "' (", objId, ")")
}
return err
}
func genId() string {
b := make([]byte, 12)
rand.Read(b)
return base64.URLEncoding.EncodeToString(b)
}

32
tools/togridfs/config.go
View file

@ -1,32 +0,0 @@
package main
const (
PORT = "8080"
DB_IP = "127.0.0.1"
DB_NAME = "trantor"
META_COLL = "meta"
BOOKS_COLL = "books"
TAGS_COLL = "tags"
USERS_COLL = "users"
STATS_COLL = "statistics"
FS_BOOKS = "fs_books"
FS_IMGS = "fs_imgs"
PASS_SALT = "ImperialLibSalt"
MINUTES_UPDATE_TAGS = 10
TAGS_DISPLAY = 50
SEARCH_ITEMS_PAGE = 20
NEW_ITEMS_PAGE = 50
TEMPLATE_PATH = "templates/"
CSS_PATH = "css/"
JS_PATH = "js/"
IMG_PATH = "img/"
IMG_WIDTH_BIG = 300
IMG_WIDTH_SMALL = 60
IMG_QUALITY = 80
STATS_CHAN_SIZE = 100
)

141
tools/togridfs/cover.go
View file

@ -1,141 +0,0 @@
package main
import _ "image/png"
import _ "image/jpeg"
import _ "image/gif"
import (
"bytes"
"git.gitorious.org/go-pkg/epubgo.git"
"github.com/nfnt/resize"
"image"
"image/jpeg"
"io"
"io/ioutil"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"log"
"regexp"
"strings"
)
func GetCover(e *epubgo.Epub, title string) (bson.ObjectId, bson.ObjectId) {
imgId, smallId := searchCommonCoverNames(e, title)
if imgId != "" {
return imgId, smallId
}
/* search for img on the text */
exp, _ := regexp.Compile("<ima?g.*[(src)(href)]=[\"']([^\"']*(\\.[^\\.\"']*))[\"']")
it, errNext := e.Spine()
for errNext == nil {
file, err := it.Open()
if err != nil {
break
}
defer file.Close()
txt, err := ioutil.ReadAll(file)
if err != nil {
break
}
res := exp.FindSubmatch(txt)
if res != nil {
href := string(res[1])
urlPart := strings.Split(it.Url(), "/")
url := strings.Join(urlPart[:len(urlPart)-1], "/")
if href[:3] == "../" {
href = href[3:]
url = strings.Join(urlPart[:len(urlPart)-2], "/")
}
href = strings.Replace(href, "%20", " ", -1)
href = strings.Replace(href, "%27", "'", -1)
href = strings.Replace(href, "%28", "(", -1)
href = strings.Replace(href, "%29", ")", -1)
if url == "" {
url = href
} else {
url = url + "/" + href
}
img, err := e.OpenFile(url)
if err == nil {
defer img.Close()
return storeImg(img, title)
}
}
errNext = it.Next()
}
return "", ""
}
func searchCommonCoverNames(e *epubgo.Epub, title string) (bson.ObjectId, bson.ObjectId) {
for _, p := range []string{"cover.jpg", "Images/cover.jpg", "cover.jpeg", "cover1.jpg", "cover1.jpeg"} {
img, err := e.OpenFile(p)
if err == nil {
defer img.Close()
return storeImg(img, title)
}
}
return "", ""
}
func storeImg(img io.Reader, title string) (bson.ObjectId, bson.ObjectId) {
/* open the files */
fBig, err := createCoverFile(title)
if err != nil {
log.Println("Error creating", title, ":", err.Error())
return "", ""
}
defer fBig.Close()
fSmall, err := createCoverFile(title + "_small")
if err != nil {
log.Println("Error creating", title+"_small", ":", err.Error())
return "", ""
}
defer fSmall.Close()
/* resize img */
var img2 bytes.Buffer
img1 := io.TeeReader(img, &img2)
jpgOptions := jpeg.Options{IMG_QUALITY}
imgResized, err := resizeImg(img1, IMG_WIDTH_BIG)
if err != nil {
log.Println("Error resizing big image:", err.Error())
return "", ""
}
err = jpeg.Encode(fBig, imgResized, &jpgOptions)
if err != nil {
log.Println("Error encoding big image:", err.Error())
return "", ""
}
imgSmallResized, err := resizeImg(&img2, IMG_WIDTH_SMALL)
if err != nil {
log.Println("Error resizing small image:", err.Error())
return "", ""
}
err = jpeg.Encode(fSmall, imgSmallResized, &jpgOptions)
if err != nil {
log.Println("Error encoding small image:", err.Error())
return "", ""
}
idBig, _ := fBig.Id().(bson.ObjectId)
idSmall, _ := fSmall.Id().(bson.ObjectId)
return idBig, idSmall
}
func createCoverFile(title string) (*mgo.GridFile, error) {
fs := db.GetFS(FS_IMGS)
return fs.Create(title + ".jpg")
}
func resizeImg(imgReader io.Reader, width uint) (image.Image, error) {
img, _, err := image.Decode(imgReader)
if err != nil {
return nil, err
}
return resize.Resize(width, 0, img, resize.NearestNeighbor), nil
}

252
tools/togridfs/database.go
View file

@ -1,252 +0,0 @@
package main
import (
"crypto/md5"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"time"
)
const (
META_TYPE_TAGS = "tags updated"
)
var db *DB
type Book struct {
Id string `bson:"_id"`
Title string
Author []string
Contributor string
Publisher string
Description string
Subject []string
Date string
Lang []string
Isbn string
Type string
Format string
Source string
Relation string
Coverage string
Rights string
Meta string
File bson.ObjectId
Cover bson.ObjectId
CoverSmall bson.ObjectId
Active bool
Keywords []string
Path string
}
type DB struct {
session *mgo.Session
meta *mgo.Collection
books *mgo.Collection
tags *mgo.Collection
user *mgo.Collection
stats *mgo.Collection
}
func initDB() *DB {
var err error
d := new(DB)
d.session, err = mgo.Dial(DB_IP)
if err != nil {
panic(err)
}
database := d.session.DB(DB_NAME)
d.meta = database.C(META_COLL)
d.books = database.C(BOOKS_COLL)
d.tags = database.C(TAGS_COLL)
d.user = database.C(USERS_COLL)
d.stats = database.C(STATS_COLL)
return d
}
func (d *DB) Close() {
d.session.Close()
}
func md5Pass(pass string) []byte {
h := md5.New()
hash := h.Sum(([]byte)(PASS_SALT + pass))
return hash
}
func (d *DB) SetPassword(user string, pass string) error {
hash := md5Pass(pass)
return d.user.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}})
}
func (d *DB) UserValid(user string, pass string) bool {
hash := md5Pass(pass)
n, err := d.user.Find(bson.M{"user": user, "pass": hash}).Count()
if err != nil {
return false
}
return n != 0
}
func (d *DB) InsertStats(stats interface{}) error {
return d.stats.Insert(stats)
}
func (d *DB) InsertBook(book interface{}) error {
return d.books.Insert(book)
}
func (d *DB) RemoveBook(id bson.ObjectId) error {
return d.books.Remove(bson.M{"_id": id})
}
func (d *DB) UpdateBook(id bson.ObjectId, data interface{}, unset ...interface{}) error {
if len(unset) > 0 {
return d.books.Update(bson.M{"_id": id}, bson.M{"$set": data, "$unset": unset[0]})
}
return d.books.Update(bson.M{"_id": id}, bson.M{"$set": data})
}
func (d *DB) IncVisit(id bson.ObjectId) error {
return d.books.Update(bson.M{"_id": id}, bson.M{"$inc": bson.M{"VisitsCount": 1}})
}
func (d *DB) IncDownload(id bson.ObjectId) error {
return d.books.Update(bson.M{"_id": id}, bson.M{"$inc": bson.M{"DownloadCount": 1}})
}
/* optional parameters: length and start index
*
* Returns: list of books, number found and err
*/
func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) {
var start, length int
if len(r) > 0 {
length = r[0]
if len(r) > 1 {
start = r[1]
}
}
q := d.books.Find(query).Sort("-_id")
num, err = q.Count()
if err != nil {
return
}
if start != 0 {
q = q.Skip(start)
}
if length != 0 {
q = q.Limit(length)
}
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
/* Get the most visited books
*/
func (d *DB) GetVisitedBooks(num int) (books []Book, err error) {
var q *mgo.Query
q = d.books.Find(bson.M{"active": true}).Sort("-VisitsCount").Limit(num)
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
/* Get the most downloaded books
*/
func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) {
var q *mgo.Query
q = d.books.Find(bson.M{"active": true}).Sort("-DownloadCount").Limit(num)
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
/* optional parameters: length and start index
*
* Returns: list of books, number found and err
*/
func (d *DB) GetNewBooks(r ...int) (books []Book, num int, err error) {
return d.GetBooks(bson.M{"$nor": []bson.M{{"active": true}}}, r...)
}
func (d *DB) BookActive(id bson.ObjectId) bool {
var book Book
err := d.books.Find(bson.M{"_id": id}).One(&book)
if err != nil {
return false
}
return book.Active
}
func (d *DB) GetFS(prefix string) *mgo.GridFS {
return d.session.DB(DB_NAME).GridFS(prefix)
}
func (d *DB) areTagsOutdated() bool {
var result struct {
Id bson.ObjectId `bson:"_id"`
}
err := d.meta.Find(bson.M{"type": META_TYPE_TAGS}).One(&result)
if err != nil {
return true
}
lastUpdate := result.Id.Time()
return time.Since(lastUpdate).Minutes() > MINUTES_UPDATE_TAGS
}
func (d *DB) updateTags() error {
_, err := d.meta.RemoveAll(bson.M{"type": META_TYPE_TAGS})
if err != nil {
return err
}
var mr mgo.MapReduce
mr.Map = "function() { " +
"if (this.active) { this.subject.forEach(function(s) { emit(s, 1); }); }" +
"}"
mr.Reduce = "function(tag, vals) { " +
"var count = 0;" +
"vals.forEach(function() { count += 1; });" +
"return count;" +
"}"
mr.Out = bson.M{"replace": TAGS_COLL}
_, err = d.books.Find(bson.M{"active": true}).MapReduce(&mr, nil)
if err != nil {
return err
}
return d.meta.Insert(bson.M{"type": META_TYPE_TAGS})
}
func (d *DB) GetTags(numTags int) ([]string, error) {
if d.areTagsOutdated() {
err := d.updateTags()
if err != nil {
return nil, err
}
}
var result []struct {
Tag string "_id"
}
err := d.tags.Find(nil).Sort("-value").Limit(numTags).All(&result)
if err != nil {
return nil, err
}
tags := make([]string, len(result))
for i, r := range result {
tags[i] = r.Tag
}
return tags, nil
}

87
tools/togridfs/togridfs.go
View file

@ -1,87 +0,0 @@
package main
import (
"fmt"
"git.gitorious.org/go-pkg/epubgo.git"
"io"
"labix.org/v2/mgo/bson"
"os"
)
func main() {
db = initDB()
defer db.Close()
books, _, _ := db.GetBooks(bson.M{})
for _, book := range books {
if book.Path == "" {
fmt.Println("don't needed -- ", book.Title)
continue
}
fmt.Println(book.Title)
path := getPath(book)
id, err := storeFile(path, book)
if err != nil {
fmt.Println("storeFile ================", err)
db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"active": false})
continue
}
cover, coverSmall, err := cover(path, book)
if err != nil {
fmt.Println("cover ================", err)
db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"active": false, "file": id})
continue
}
if cover != "" {
db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"cover": cover, "coversmall": coverSmall, "file": id}, bson.M{"path": 1})
} else {
fmt.Println("No cover ================", book.Title)
db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"active": false, "file": id})
}
}
}
func getPath(book Book) string {
if !book.Active {
return "new/" + book.Path
}
return "books/" + book.Path
}
func storeFile(path string, book Book) (bson.ObjectId, error) {
fs := db.GetFS(FS_BOOKS)
file, err := os.Open(path)
if err != nil {
return "", err
}
defer file.Close()
fw, err := fs.Create(book.Title + ".epub")
if err != nil {
return "", err
}
defer fw.Close()
id, _ := fw.Id().(bson.ObjectId)
_, err = io.Copy(fw, file)
if err != nil {
return id, err
}
return id, nil
}
func cover(path string, book Book) (bson.ObjectId, bson.ObjectId, error) {
e, err := epubgo.Open(path)
if err != nil {
return "", "", err
}
defer e.Close()
cover, coverSmall := GetCover(e, book.Title)
return cover, coverSmall, err
}

22
tools/update/config.go
View file

@ -1,22 +0,0 @@
package main
const (
PORT = "8080"
DB_IP = "127.0.0.1"
DB_NAME = "trantor"
BOOKS_COLL = "books"
NEW_BOOKS_COLL = "new"
USERS_COLL = "users"
PASS_SALT = "ImperialLibSalt"
TAGS_DISPLAY = 50
SEARCH_ITEMS_PAGE = 10
TEMPLATE_PATH = "templates/"
BOOKS_PATH = "books/"
COVER_PATH = "cover/"
NEW_PATH = "new/"
CSS_PATH = "css/"
JS_PATH = "js/"
IMG_PATH = "img/"
RESIZE_CMD = "/usr/bin/convert -resize 300 -quality 60 "
RESIZE_THUMB_CMD = "/usr/bin/convert -resize 60 -quality 60 "
)

214
tools/update/database.go
View file

@ -1,214 +0,0 @@
package main
import (
"crypto/md5"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"sort"
)
var db *DB
type Book struct {
Id string `bson:"_id"`
Title string
Author []string
Contributor string
Publisher string
Description string
Subject []string
Date string
Lang []string
Type string
Format string
Source string
Relation string
Coverage string
Rights string
Meta string
Path string
Cover string
CoverSmall string
Active bool
Keywords []string
}
type DB struct {
session *mgo.Session
books *mgo.Collection
user *mgo.Collection
}
func initDB() *DB {
var err error
d := new(DB)
d.session, err = mgo.Dial(DB_IP)
if err != nil {
panic(err)
}
d.books = d.session.DB(DB_NAME).C(BOOKS_COLL)
d.user = d.session.DB(DB_NAME).C(USERS_COLL)
return d
}
func (d *DB) Close() {
d.session.Close()
}
func md5Pass(pass string) []byte {
h := md5.New()
hash := h.Sum(([]byte)(PASS_SALT + pass))
return hash
}
func (d *DB) SetPassword(user string, pass string) error {
hash := md5Pass(pass)
return d.user.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}})
}
func (d *DB) UserValid(user string, pass string) bool {
hash := md5Pass(pass)
n, err := d.user.Find(bson.M{"user": user, "pass": hash}).Count()
if err != nil {
return false
}
return n != 0
}
func (d *DB) InsertBook(book interface{}) error {
return d.books.Insert(book)
}
func (d *DB) RemoveBook(id bson.ObjectId) error {
return d.books.Remove(bson.M{"_id": id})
}
func (d *DB) UpdateBook(id bson.ObjectId, data interface{}) error {
return d.books.Update(bson.M{"_id": id}, bson.M{"$set": data})
}
func (d *DB) IncVisit(id bson.ObjectId) error {
return d.books.Update(bson.M{"_id": id}, bson.M{"$inc": bson.M{"VisitsCount": 1}})
}
func (d *DB) IncDownload(path string) error {
return d.books.Update(bson.M{"path": path}, bson.M{"$inc": bson.M{"DownloadCount": 1}})
}
/* optional parameters: length and start index
*
* Returns: list of books, number found and err
*/
func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) {
var start, length int
if len(r) > 0 {
length = r[0]
if len(r) > 1 {
start = r[1]
}
}
q := d.books.Find(query).Sort("-_id")
num, err = q.Count()
if err != nil {
return
}
if start != 0 {
q = q.Skip(start)
}
if length != 0 {
q = q.Limit(length)
}
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
/* Get the most visited books
*/
func (d *DB) GetVisitedBooks(num int) (books []Book, err error) {
var q *mgo.Query
q = d.books.Find(bson.M{"active": true}).Sort("-VisitsCount").Limit(num)
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
/* Get the most downloaded books
*/
func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) {
var q *mgo.Query
q = d.books.Find(bson.M{"active": true}).Sort("-DownloadCount").Limit(num)
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
/* Returns: list of books, number found and err
*/
func (d *DB) GetNewBooks() (books []Book, num int, err error) {
var q *mgo.Query
q = d.books.Find(bson.M{"$nor": []bson.M{{"active": true}}}).Sort("-_id")
num, err = q.Count()
if err != nil {
return
}
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
func (d *DB) BookActive(id bson.ObjectId) bool {
var book Book
err := d.books.Find(bson.M{"_id": id}).One(&book)
if err != nil {
return false
}
return book.Active
}
type tagsList []struct {
Subject string "_id"
Count int "value"
}
func (t tagsList) Len() int {
return len(t)
}
func (t tagsList) Less(i, j int) bool {
return t[i].Count > t[j].Count
}
func (t tagsList) Swap(i, j int) {
aux := t[i]
t[i] = t[j]
t[j] = aux
}
func (d *DB) GetTags() (tagsList, error) {
// TODO: cache the tags
var mr mgo.MapReduce
mr.Map = "function() { " +
"if (this.active) { this.subject.forEach(function(s) { emit(s, 1); }); }" +
"}"
mr.Reduce = "function(tag, vals) { " +
"var count = 0;" +
"vals.forEach(function() { count += 1; });" +
"return count;" +
"}"
var result tagsList
_, err := d.books.Find(nil).MapReduce(&mr, &result)
if err == nil {
sort.Sort(result)
}
return result, err
}

265
tools/update/store.go
View file

@ -1,265 +0,0 @@
package main
import (
"git.gitorious.org/go-pkg/epub.git"
"io"
"log"
"os"
"os/exec"
"regexp"
"strconv"
"strings"
"unicode/utf8"
)
func ParseFile(path string) (string, error) {
book := map[string]interface{}{}
e, err := epub.Open(NEW_PATH+path, 0)
if err != nil {
return "", err
}
defer e.Close()
title := cleanStr(strings.Join(e.Metadata(epub.EPUB_TITLE), ", "))
book["title"] = title
book["author"] = parseAuthr(e.Metadata(epub.EPUB_CREATOR))
book["contributor"] = cleanStr(strings.Join(e.Metadata(epub.EPUB_CONTRIB), ", "))
book["publisher"] = cleanStr(strings.Join(e.Metadata(epub.EPUB_PUBLISHER), ", "))
book["description"] = parseDescription(e.Metadata(epub.EPUB_DESCRIPTION))
book["subject"] = parseSubject(e.Metadata(epub.EPUB_SUBJECT))
book["date"] = parseDate(e.Metadata(epub.EPUB_DATE))
book["lang"] = e.Metadata(epub.EPUB_LANG)
book["type"] = strings.Join(e.Metadata(epub.EPUB_TYPE), ", ")
book["format"] = strings.Join(e.Metadata(epub.EPUB_FORMAT), ", ")
book["source"] = strings.Join(e.Metadata(epub.EPUB_SOURCE), ", ")
book["relation"] = strings.Join(e.Metadata(epub.EPUB_RELATION), ", ")
book["coverage"] = strings.Join(e.Metadata(epub.EPUB_COVERAGE), ", ")
book["rights"] = strings.Join(e.Metadata(epub.EPUB_RIGHTS), ", ")
book["meta"] = strings.Join(e.Metadata(epub.EPUB_META), ", ")
book["path"] = path
cover, coverSmall := getCover(e, title)
book["cover"] = cover
book["coversmall"] = coverSmall
book["keywords"] = keywords(book)
db.InsertBook(book)
return title, nil
}
func StoreNewFile(name string, file io.Reader) (string, error) {
path := storePath(name)
fw, err := os.Create(NEW_PATH + path)
if err != nil {
return "", err
}
defer fw.Close()
const size = 1024
var n int = size
buff := make([]byte, size)
for n == size {
n, err = file.Read(buff)
fw.Write(buff)
}
return path, nil
}
func StoreBook(book Book) (path string, err error) {
title := book.Title
path = validFileName(BOOKS_PATH, title, ".epub")
oldPath := NEW_PATH + book.Path
r, _ := utf8.DecodeRuneInString(title)
folder := string(r)
if _, err = os.Stat(BOOKS_PATH + folder); err != nil {
err = os.Mkdir(BOOKS_PATH+folder, os.ModePerm)
if err != nil {
log.Println("Error creating", BOOKS_PATH+folder, ":", err.Error())
return
}
}
cmd := exec.Command("mv", oldPath, BOOKS_PATH+path)
err = cmd.Run()
return
}
func DeleteBook(book Book) {
if book.Cover != "" {
os.RemoveAll(book.Cover[1:])
}
if book.CoverSmall != "" {
os.RemoveAll(book.CoverSmall[1:])
}
os.RemoveAll(book.Path)
}
func validFileName(path string, title string, extension string) string {
title = strings.Replace(title, "/", "_", -1)
title = strings.Replace(title, "?", "_", -1)
title = strings.Replace(title, "#", "_", -1)
r, _ := utf8.DecodeRuneInString(title)
folder := string(r)
file := folder + "/" + title + extension
_, err := os.Stat(path + file)
for i := 0; err == nil; i++ {
file = folder + "/" + title + "_" + strconv.Itoa(i) + extension
_, err = os.Stat(path + file)
}
return file
}
func storePath(name string) string {
path := name
_, err := os.Stat(NEW_PATH + path)
for i := 0; err == nil; i++ {
path = strconv.Itoa(i) + "_" + name
_, err = os.Stat(NEW_PATH + path)
}
return path
}
func cleanStr(str string) string {
str = strings.Replace(str, "&#39;", "'", -1)
exp, _ := regexp.Compile("&[^;]*;")
str = exp.ReplaceAllString(str, "")
exp, _ = regexp.Compile("[ ,]*$")
str = exp.ReplaceAllString(str, "")
return str
}
func storeImg(img []byte, title, extension string) (string, string) {
r, _ := utf8.DecodeRuneInString(title)
folder := string(r)
if _, err := os.Stat(COVER_PATH + folder); err != nil {
err = os.Mkdir(COVER_PATH+folder, os.ModePerm)
if err != nil {
log.Println("Error creating", COVER_PATH+folder, ":", err.Error())
return "", ""
}
}
imgPath := validFileName(COVER_PATH, title, extension)
/* store img on disk */
file, err := os.Create(COVER_PATH + imgPath)
if err != nil {
log.Println("Error creating", COVER_PATH+imgPath, ":", err.Error())
return "", ""
}
defer file.Close()
file.Write(img)
/* resize img */
resize := append(strings.Split(RESIZE_CMD, " "), COVER_PATH+imgPath, COVER_PATH+imgPath)
cmd := exec.Command(resize[0], resize[1:]...)
cmd.Run()
imgPathSmall := validFileName(COVER_PATH, title, "_small"+extension)
resize = append(strings.Split(RESIZE_THUMB_CMD, " "), COVER_PATH+imgPath, COVER_PATH+imgPathSmall)
cmd = exec.Command(resize[0], resize[1:]...)
cmd.Run()
return imgPath, imgPathSmall
}
func getCover(e *epub.Epub, title string) (string, string) {
/* Try first common names */
for _, p := range []string{"cover.jpg", "Images/cover.jpg", "cover.jpeg", "cover1.jpg", "cover1.jpeg"} {
img := e.Data(p)
if len(img) != 0 {
return storeImg(img, title, ".jpg")
}
}
/* search for img on the text */
exp, _ := regexp.Compile("<ima?g.*[(src)(href)]=[\"']([^\"']*(\\.[^\\.\"']*))[\"']")
it := e.Iterator(epub.EITERATOR_SPINE)
defer it.Close()
var err error = nil
txt := it.Curr()
for err == nil {
res := exp.FindStringSubmatch(txt)
if res != nil {
urlPart := strings.Split(it.CurrUrl(), "/")
url := strings.Join(urlPart[:len(urlPart)-1], "/")
if res[1][:3] == "../" {
res[1] = res[1][3:]
url = strings.Join(urlPart[:len(urlPart)-2], "/")
}
res[1] = strings.Replace(res[1], "%20", " ", -1)
res[1] = strings.Replace(res[1], "%27", "'", -1)
res[1] = strings.Replace(res[1], "%28", "(", -1)
res[1] = strings.Replace(res[1], "%29", ")", -1)
if url == "" {
url = res[1]
} else {
url = url + "/" + res[1]
}
img := e.Data(url)
if len(img) != 0 {
return storeImg(img, title, res[2])
}
}
txt, err = it.Next()
}
return "", ""
}
func parseAuthr(creator []string) []string {
exp1, _ := regexp.Compile("^(.*\\( *([^\\)]*) *\\))*$")
exp2, _ := regexp.Compile("^[^:]*: *(.*)$")
res := make([]string, len(creator))
for i, s := range creator {
auth := exp1.FindStringSubmatch(s)
if auth != nil {
res[i] = cleanStr(strings.Join(auth[2:], ", "))
} else {
auth := exp2.FindStringSubmatch(s)
if auth != nil {
res[i] = cleanStr(auth[1])
} else {
res[i] = cleanStr(s)
}
}
}
return res
}
func parseDescription(description []string) string {
str := cleanStr(strings.Join(description, ", "))
exp, _ := regexp.Compile("<[^>]*>")
str = exp.ReplaceAllString(str, "")
str = strings.Replace(str, "&amp;", "&", -1)
str = strings.Replace(str, "&lt;", "<", -1)
str = strings.Replace(str, "&gt;", ">", -1)
str = strings.Replace(str, "\\n", "\n", -1)
return str
}
func parseSubject(subject []string) []string {
var res []string
for _, s := range subject {
res = append(res, strings.Split(s, " / ")...)
}
return res
}
func parseDate(date []string) string {
if len(date) == 0 {
return ""
}
return strings.Replace(date[0], "Unspecified: ", "", -1)
}
func keywords(b map[string]interface{}) (k []string) {
title, _ := b["title"].(string)
k = strings.Split(title, " ")
author, _ := b["author"].([]string)
for _, a := range author {
k = append(k, strings.Split(a, " ")...)
}
publisher, _ := b["publisher"].(string)
k = append(k, strings.Split(publisher, " ")...)
subject, _ := b["subject"].([]string)
k = append(k, subject...)
return
}

27
tools/update/update.go
View file

@ -1,27 +0,0 @@
package main
import (
"fmt"
"git.gitorious.org/go-pkg/epub.git"
"labix.org/v2/mgo/bson"
)
func main() {
db = initDB()
defer db.Close()
books, _, _ := db.GetBooks(bson.M{})
for _, book := range books {
fmt.Println(book.Title)
e, err := epub.Open(BOOKS_PATH+book.Path, 0)
if err != nil {
fmt.Println("================", err)
}
cover, coverSmall := getCover(e, book.Title)
if cover != "" {
db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"cover": cover, "coversmall": coverSmall})
}
e.Close()
}
}