archives/trantor
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[WIP] migration to psql

Browse source 
TODO:
[ ] stats
[ ] indexes
This commit is contained in:
Las Zenow 2016-07-30 07:10:33 -04:00
parent e1bd235785
commit e72de38725
24 changed files with 648 additions and 936 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

85
createdb.sql Normal file
View file

@ -0,0 +1,85 @@
CREATE TABLE books (
id varchar(16) primary key,
title text,
authors text[],
contributor text,
publisher text,
description text,
tags text[],
date text,
lang varchar(3),
isbn varchar(13),
file_size integer,
cover boolean,
active boolean,
upload_date timestamp,
tsv tsvector
);
--
-- Books text search index
--
CREATE FUNCTION books_trigger() RETURNS trigger AS $$
declare
lang_config regconfig;
begin
lang_config := 'simple';
if new.lang = 'da' then
lang_config := 'danish';
elsif new.lang = 'nl' then
lang_config := 'dutch';
elsif new.lang = 'en' then
lang_config := 'english';
elsif new.lang = 'fi' then
lang_config := 'finnish';
elsif new.lang = 'fr' then
lang_config := 'french';
elsif new.lang = 'de' then
lang_config :='german';
elsif new.lang = 'hu' then
lang_config :='hungarian';
elsif new.lang = 'it' then
lang_config :='italian';
elsif new.lang = 'no' then
lang_config :='norwegian';
elsif new.lang = 'pt' then
lang_config :='portuguese';
elsif new.lang = 'ro' then
lang_config :='romanian';
elsif new.lang = 'ru' then
lang_config :='russian';
elsif new.lang = 'es' then
lang_config :='spanish';
elsif new.lang = 'sv' then
lang_config :='swedish';
elsif new.lang = 'tr' then
lang_config :='turkish';
end if;
new.tsv :=
setweight(to_tsvector(lang_config, coalesce(new.title,'')), 'A') ||
setweight(to_tsvector('simple', coalesce(array_to_string(new.authors, ' '),'')), 'A') ||
setweight(to_tsvector('simple', coalesce(new.contributor,'')), 'B') ||
setweight(to_tsvector('simple', coalesce(new.publisher,'')), 'B') ||
setweight(to_tsvector(lang_config, coalesce(array_to_string(new.tags, ' '),'')), 'C') ||
setweight(to_tsvector(lang_config, coalesce(new.description,'')), 'D');
return new;
end
$$ LANGUAGE plpgsql;
CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE
ON books FOR EACH ROW EXECUTE PROCEDURE books_trigger();
CREATE INDEX books_idx ON books USING GIN (tsv);
CREATE TABLE news (
id serial unique,
date time,
text text
);
CREATE TABLE users (
id serial unique,
username varchar(255) unique,
password bytea,
salt bytea,
role varchar(255)
);

346
lib/database/books.go
View file

@ -1,238 +1,210 @@
package database package database
import ( import (
log "github.com/cihub/seelog"
"strings" "strings"
"time" "time"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
const (
books_coll = "books"
) )
// TODO: Author -> Authors, Subject -> Tags
type Book struct { type Book struct {
Id string Id string
Title string Title string
Author []string Author []string `sql:"authors" pg:",array"`
Contributor string Contributor string
Publisher string Publisher string
Description string Description string
Subject []string Subject []string `sql:"tags" pg:",array"`
Date string Date string
Lang []string Lang string
Isbn string Isbn string
Type string FileSize int
Format string Cover bool
Source string Active bool
Relation string UploadDate time.Time
Coverage string Tsv string
Rights string //BadQuality int `bad_quality`
Meta string BadQualityReporters []string `sql:"-"` // TODO: deprecate??
FileSize int
Cover bool
Active bool
BadQuality int `bad_quality`
BadQualityReporters []string `bad_quality_reporters`
} }
type history struct { // TODO: missing history
Date time.Time
Changes bson.M
}
func indexBooks(coll *mgo.Collection) { // AddBook to the database
indexes := []mgo.Index{ func (db *pgDB) AddBook(book Book) error {
{ emptyTime := time.Time{}
Key: []string{"id"}, if book.UploadDate == emptyTime {
Unique: true, book.UploadDate = time.Now()
Background: true,
},
{
Key: []string{"active", "-_id"},
Background: true,
},
{
Key: []string{"active", "-bad_quality", "-_id"},
Background: true,
},
{
Key: []string{"$text:title", "$text:author", "$text:contributor",
"$text:publisher", "$text:subject", "$text:description"},
Weights: map[string]int{"title": 20, "author": 20, "contributor": 15,
"publisher": 15, "subject": 10, "description": 5},
LanguageOverride: "_lang",
Background: true,
},
}
for _, k := range []string{"lang", "title", "author", "subject"} {
idx := mgo.Index{
Key: []string{"active", k, "-_id"},
Background: true,
}
indexes = append(indexes, idx)
} }
for _, idx := range indexes { return db.sql.Create(&book)
err := coll.EnsureIndex(idx)
if err != nil {
log.Error("Error indexing books: ", err)
}
}
} }
func addBook(coll *mgo.Collection, book map[string]interface{}) error { // GetBooks matching query
book["_lang"] = metadataLang(book) func (db *pgDB) GetBooks(query string, length int, start int) (books []Book, num int, err error) {
return coll.Insert(book) return db.getBooks(true, query, length, start)
} }
func getBooks(coll *mgo.Collection, query string, length int, start int) (books []Book, num int, err error) { // TODO: func (db *pgDB) GetBooksIter() Iter {
q := buildQuery(query)
q["active"] = true func (db *pgDB) GetNewBooks(query string, length int, start int) (books []Book, num int, err error) {
return _getBooks(coll, q, length, start) return db.getBooks(false, query, length, start)
} }
func getNewBooks(coll *mgo.Collection, query string, length int, start int) (books []Book, num int, err error) { func (db *pgDB) getBooks(active bool, query string, length int, start int) (books []Book, num int, err error) {
q := buildQuery(query) sqlQuery := db.sql.Model(&books)
q["$nor"] = []bson.M{{"active": true}}
return _getBooks(coll, q, length, start)
}
func getBooksIter(coll *mgo.Collection) Iter { searchCondition := "active = "
return coll.Find(bson.M{}).Iter() if active {
} searchCondition = "true"
} else {
func _getBooks(coll *mgo.Collection, query bson.M, length int, start int) (books []Book, num int, err error) { searchCondition = "false"
q := getBookQuery(coll, query)
num, err = q.Count()
if err != nil {
return
}
if start != 0 {
q = q.Skip(start)
}
if length != 0 {
q = q.Limit(length)
} }
err = q.All(&books) params := []interface{}{}
return textQuery, columnQuerys := buildQuery(query)
} for _, c := range columnQuerys {
searchCondition = searchCondition + " AND " + c.column + " ILIKE ?"
func getBookQuery(coll *mgo.Collection, query bson.M) *mgo.Query { params = append(params, c.value)
sort := []string{"$textScore:score"}
if _, present := query["bad_quality"]; present {
sort = append(sort, "-bad_quality")
} }
sort = append(sort, "-_id") if textQuery != "" {
searchCondition = searchCondition + " AND to_tsquery(?) @@ tsv"
params = append(params, textQuery)
}
sqlQuery = sqlQuery.Where(searchCondition, params...)
return coll.Find(query).Select(bson.M{"score": bson.M{"$meta": "textScore"}}).Sort(sort...) if textQuery != "" {
sqlQuery = sqlQuery.Order("ts_rank(tsv, to_tsquery(?)) DESC, upload_date DESC", textQuery)
} else {
sqlQuery = sqlQuery.Order("upload_date DESC")
}
num, err = sqlQuery.
Offset(start).
Limit(length).
SelectAndCountEstimate(100)
return books, num, err
} }
func getBookId(coll *mgo.Collection, id string) (Book, error) { func (db *pgDB) GetBookId(id string) (Book, error) {
var book Book var book Book
err := coll.Find(bson.M{"id": id}).One(&book) err := db.sql.Model(&book).
Where("id = ?", id).
Select()
return book, err return book, err
} }
func deleteBook(coll *mgo.Collection, id string) error { func (db *pgDB) DeleteBook(id string) error {
return coll.Remove(bson.M{"id": id}) _, err := db.sql.Model(&Book{}).
Where("id = ?", id).
Delete()
return err
} }
func updateBook(coll *mgo.Collection, id string, data map[string]interface{}) error { func (db *pgDB) UpdateBook(id string, data map[string]interface{}) error {
var book map[string]interface{} setCondition := ""
record := history{time.Now(), bson.M{}} params := []interface{}{}
for col, val := range data {
err := coll.Find(bson.M{"id": id}).One(&book) colValid := false
if err != nil { for _, name := range []string{"title", "authors", "contributor", "publisher",
return err "description", "tags", "date", "lang", "isbn"} {
} if col == name {
for k, _ := range data { colValid = true
record.Changes[k] = book[k] break
if k == "lang" { }
data["_lang"] = metadataLang(data)
} }
} if !colValid {
continue
return coll.Update(bson.M{"id": id}, bson.M{"$set": data, "$push": bson.M{"history": record}})
}
func flagBadQuality(coll *mgo.Collection, id string, user string) error {
b, err := getBookId(coll, id)
if err != nil {
return err
}
for _, reporter := range b.BadQualityReporters {
if reporter == user {
return nil
} }
if len(setCondition) != 0 {
setCondition += ", "
}
setCondition += col + " = ?"
params = append(params, val)
} }
return coll.Update( _, err := db.sql.Model(&Book{}).
bson.M{"id": id}, Set(setCondition, params...).
bson.M{ Where("id = ?", id).
"$inc": bson.M{"bad_quality": 1}, Update()
"$addToSet": bson.M{"bad_quality_reporters": user}, return err
},
)
} }
func activeBook(coll *mgo.Collection, id string) error { func (db *pgDB) FlagBadQuality(id string, user string) error {
data := map[string]interface{}{"active": true} // TODO: delete me
return coll.Update(bson.M{"id": id}, bson.M{"$set": data}) return nil
} }
func isBookActive(coll *mgo.Collection, id string) bool { func (db *pgDB) ActiveBook(id string) error {
var book Book uploadDate := time.Now()
err := coll.Find(bson.M{"id": id}).One(&book) _, err := db.sql.Model(&Book{}).
if err != nil { Set("active = true, upload_date = ? ", uploadDate).
Where("id = ?", id).
Update()
return err
}
func (db *pgDB) IsBookActive(id string) bool {
var active []bool
err := db.sql.Model(&Book{}).
Column("active").
Where("id = ?", id).
Select(&active)
if err != nil || len(active) != 1 {
return false return false
} }
return book.Active return active[0]
} }
func buildQuery(q string) bson.M { type columnq struct {
text := "" column string
query := bson.M{} value string
words := strings.Split(q, " ") }
func buildQuery(query string) (string, []columnq) {
textQuery := ""
columnQuerys := []columnq{}
words := strings.Split(query, " ")
for _, w := range words { for _, w := range words {
if w == "" {
continue
}
tag := strings.SplitN(w, ":", 2) tag := strings.SplitN(w, ":", 2)
if len(tag) > 1 { if len(tag) > 1 && tag[1] != "" {
if tag[0] == "flag" { value := strings.Replace(tag[1], "%", "\\%", 0)
query[tag[1]] = bson.M{"$gt": 0} value = strings.Replace(value, "_", "\\_", 0)
} else { expr := "%" + value + "%"
query[tag[0]] = bson.RegEx{tag[1], "i"} //FIXME: this should be a list switch tag[0] {
case "lang":
columnQuerys = append(columnQuerys, columnq{"lang", value})
case "author":
columnQuerys = append(columnQuerys, columnq{"array_to_string(authors, ' ')", expr})
case "title":
columnQuerys = append(columnQuerys, columnq{"title", expr})
case "contributor":
columnQuerys = append(columnQuerys, columnq{"contributor", expr})
case "publisher":
columnQuerys = append(columnQuerys, columnq{"publisher", expr})
case "subject":
expr = strings.ToLower(expr)
columnQuerys = append(columnQuerys, columnq{"array_to_string(tags, ' ')", expr})
case "tag":
expr = strings.ToLower(expr)
columnQuerys = append(columnQuerys, columnq{"array_to_string(tag, ' ')", expr})
case "isbn":
columnQuerys = append(columnQuerys, columnq{"isbn", expr})
case "description":
columnQuerys = append(columnQuerys, columnq{"description", expr})
} }
} else { } else {
if len(text) != 0 { if len(textQuery) != 0 {
text += " " lastChar := textQuery[len(textQuery)-1:]
if w != "&" && w != "|" && lastChar != "&" && lastChar != "|" {
textQuery += " | "
} else {
textQuery += " "
}
} }
text += w textQuery += w
} }
} }
if len(text) > 0 { return textQuery, columnQuerys
query["$text"] = bson.M{"$search": text}
}
return query
}
func metadataLang(book map[string]interface{}) string {
text_search_langs := map[string]bool{
"da": true, "nl": true, "en": true, "fi": true, "fr": true, "de": true,
"hu": true, "it": true, "nb": true, "pt": true, "ro": true, "ru": true,
"es": true, "sv": true, "tr": true}
lang, ok := book["lang"].([]string)
if !ok || len(lang) == 0 || len(lang[0]) < 2 {
return "none"
}
lcode := strings.ToLower(lang[0][0:2])
if text_search_langs[lcode] {
return lcode
}
return "none"
} }

123
lib/database/books_test.go
View file

@ -2,38 +2,50 @@ package database
import "testing" import "testing"
var book = map[string]interface{}{ var book = Book{
"title": "some title", Id: "r_m-IOzzIbA6QK5w",
"author": []string{"Alice", "Bob"}, Title: "some title",
"id": "r_m-IOzzIbA6QK5w", Author: []string{"Alice", "Bob"},
} }
func TestAddBook(t *testing.T) { func TestAddAndDeleteBook(t *testing.T) {
db := Init(test_host, test_coll) db, dbclose := testDbInit(t)
defer del(db) defer dbclose()
tAddBook(t, db) testAddBook(t, db)
books, num, err := db.GetNewBooks("", 1, 0) books, num, err := db.GetNewBooks("", 1, 0)
if err != nil { if err != nil {
t.Fatal("db.GetBooks() return an error: ", err) t.Fatal("db.GetNewBooks() return an error: ", err)
} }
if num < 1 { if num < 1 {
t.Fatalf("db.GetBooks() didn't find any result.") t.Fatalf("db.GetNewBooks() didn't find any result.")
} }
if len(books) < 1 { if len(books) < 1 {
t.Fatalf("db.GetBooks() didn't return any result.") t.Fatalf("db.GetNewBooks() didn't return any result.")
} }
if books[0].Title != book["title"] { if books[0].Title != book.Title {
t.Error("Book title don't match : '", books[0].Title, "' <=> '", book["title"], "'") t.Error("Book title don't match : '", books[0].Title, "' <=> '", book.Title, "'")
}
err = db.DeleteBook(books[0].Id)
if err != nil {
t.Fatal("db.DeleteBook() return an error: ", err)
}
books, num, err = db.GetNewBooks("", 1, 0)
if err != nil {
t.Fatal("db.GetNewBooks() return an error after delete: ", err)
}
if num != 0 {
t.Fatalf("the book was not deleted.")
} }
} }
func TestActiveBook(t *testing.T) { func TestActiveBook(t *testing.T) {
db := Init(test_host, test_coll) db, dbclose := testDbInit(t)
defer del(db) defer dbclose()
tAddBook(t, db) testAddBook(t, db)
books, _, _ := db.GetNewBooks("", 1, 0) books, _, _ := db.GetNewBooks("", 1, 0)
id := books[0].Id id := books[0].Id
@ -46,58 +58,57 @@ func TestActiveBook(t *testing.T) {
if err != nil { if err != nil {
t.Fatal("db.GetBookId(", id, ") return an error: ", err) t.Fatal("db.GetBookId(", id, ") return an error: ", err)
} }
if !b.Active {
t.Error("Book is not activated")
}
if b.Author[0] != books[0].Author[0] { if b.Author[0] != books[0].Author[0] {
t.Error("Book author don't match : '", b.Author, "' <=> '", book["author"], "'") t.Error("Book author don't match : '", b.Author, "' <=> '", book.Author, "'")
}
bs, num, err := db.GetBooks(book.Title, 20, 0)
if err != nil {
t.Fatal("db.GetBooks(", book.Title, ") return an error: ", err)
}
if num != 1 || len(bs) != 1 {
t.Fatal("We got a un expected number of books: ", num, bs)
}
if bs[0].Author[0] != book.Author[0] {
t.Error("Book author don't match : '", bs[0].Author, "' <=> '", book.Author, "'")
}
bs, num, err = db.GetBooks("none", 20, 0)
if err != nil {
t.Fatal("db.GetBooks(none) return an error: ", err)
}
if num != 0 || len(bs) != 0 {
t.Error("We got books: ", num, bs)
} }
} }
func TestFlag(t *testing.T) { func TestUpdateBook(t *testing.T) {
db := Init(test_host, test_coll) db, dbclose := testDbInit(t)
defer del(db) defer dbclose()
tAddBook(t, db) testAddBook(t, db)
id, _ := book["id"].(string)
db.ActiveBook(id) newTitle := "other title"
id2 := "tfgrBvd2ps_K4iYt" err := db.UpdateBook(book.Id, map[string]interface{}{
b2 := book "title": newTitle,
b2["id"] = id2 })
err := db.AddBook(b2)
if err != nil { if err != nil {
t.Error("db.AddBook(", book, ") return an error:", err) t.Fatal("db.UpdateBook() return an error: ", err)
}
db.ActiveBook(id2)
id3 := "tfgrBvd2ps_K4iY2"
b3 := book
b3["id"] = id3
err = db.AddBook(b3)
if err != nil {
t.Error("db.AddBook(", book, ") return an error:", err)
}
db.ActiveBook(id3)
db.FlagBadQuality(id, "1")
db.FlagBadQuality(id, "2")
db.FlagBadQuality(id3, "1")
b, _ := db.GetBookId(id)
if b.BadQuality != 2 {
t.Error("The bad quality flag was not increased")
}
b, _ = db.GetBookId(id3)
if b.BadQuality != 1 {
t.Error("The bad quality flag was not increased")
} }
books, _, _ := db.GetBooks("flag:bad_quality", 2, 0) books, num, err := db.GetNewBooks("", 1, 0)
if len(books) != 2 { if err != nil || num != 1 || len(books) != 1 {
t.Fatal("Not the right number of results to the flag search:", len(books)) t.Fatal("db.GetNewBooks() return an error: ", err)
} }
if books[0].Id != id { if books[0].Title != newTitle {
t.Error("Search for flag bad_quality is not sort right") t.Error("Book title don't match : '", books[0].Title, "' <=> '", newTitle, "'")
} }
} }
func tAddBook(t *testing.T, db DB) { func testAddBook(t *testing.T, db DB) {
err := db.AddBook(book) err := db.AddBook(book)
if err != nil { if err != nil {
t.Error("db.AddBook(", book, ") return an error:", err) t.Error("db.AddBook(", book, ") return an error:", err)

57
lib/database/database.go
View file

@ -1,19 +1,13 @@
package database package database
import ( import (
log "github.com/cihub/seelog" "gopkg.in/pg.v4"
"os"
"gopkg.in/mgo.v2"
) )
type DB interface { type DB interface {
Close() Close() error
Copy() DB AddBook(book Book) error
AddBook(book map[string]interface{}) error
GetBooks(query string, length int, start int) (books []Book, num int, err error) GetBooks(query string, length int, start int) (books []Book, num int, err error)
GetBooksIter() Iter
GetNewBooks(query string, length int, start int) (books []Book, num int, err error) GetNewBooks(query string, length int, start int) (books []Book, num int, err error)
GetBookId(id string) (Book, error) GetBookId(id string) (Book, error)
DeleteBook(id string) error DeleteBook(id string) error
@ -21,10 +15,12 @@ type DB interface {
FlagBadQuality(id string, user string) error FlagBadQuality(id string, user string) error
ActiveBook(id string) error ActiveBook(id string) error
IsBookActive(id string) bool IsBookActive(id string) bool
User(name string) *User
AddUser(name string, pass string) error AddUser(name string, pass string) error
GetRole(name string) (string, error)
SetPassword(name string, pass string) error
ValidPassword(name string, pass string) bool
AddNews(text string) error AddNews(text string) error
GetNews(num int, days int) (news []News, err error) GetNews(num int, days int) (news []New, err error)
AddStats(stats interface{}) error AddStats(stats interface{}) error
GetVisitedBooks() (books []Book, err error) GetVisitedBooks() (books []Book, err error)
UpdateMostVisited() error UpdateMostVisited() error
@ -41,22 +37,33 @@ type DB interface {
UpdateMonthDownloads() error UpdateMonthDownloads() error
} }
type Iter interface { type pgDB struct {
Close() error sql *pg.DB
Next(interface{}) bool
} }
func Init(host string, name string) DB { // Options for the database
var err error type Options struct {
db := new(mgoDB) Addr string
db.session, err = mgo.Dial(host) User string
if err != nil { Password string
log.Critical(err) Name string
os.Exit(1) }
}
db.name = name // Init the database connection
db.initIndexes() func Init(options Options) (DB, error) {
return db sql := pg.Connect(&pg.Options{
Addr: options.Addr,
User: options.User,
Password: options.Password,
Database: options.Name,
})
// TODO: create db
return &pgDB{sql}, nil
}
// Close the database connection
func (db pgDB) Close() error {
return db.sql.Close()
} }
func RO(db DB) DB { func RO(db DB) DB {

77
lib/database/database_test.go
View file

@ -1,24 +1,71 @@
package database package database
import ( import (
"io/ioutil"
"testing" "testing"
mgo "gopkg.in/mgo.v2"
) )
const ( func testDbInit(t *testing.T) (DB, func()) {
test_coll = "test_trantor" db, err := Init(Options{
test_host = "127.0.0.1" Name: "test_trantor",
) // TODO: can it be done with a local user?
User: "trantor",
Password: "trantor",
})
if err != nil {
t.Fatal("Init() return an error: ", err)
}
pgdb, _ := db.(*pgDB)
buf, err := ioutil.ReadFile("../../createdb.sql")
if err != nil {
t.Fatal("error reading sql schema: ", err)
}
schema := string(buf)
_, err = pgdb.sql.Exec(schema)
if err != nil {
t.Fatal("error setting up sql schema: ", err)
}
cleanFn := func() {
entities := []struct {
name string
query string
}{
{"table", "select tablename from pg_tables where schemaname = 'public'"},
{"index", "select indexname from pg_indexes where schemaname = 'public'"},
{"function", `SELECT format('%s(%s)', p.proname, pg_get_function_identity_arguments(p.oid))
FROM pg_catalog.pg_namespace n
JOIN pg_catalog.pg_proc p
ON p.pronamespace = n.oid
WHERE n.nspname = 'public'`},
{"trigger", "select tgname from pg_trigger"},
}
for _, entity := range entities {
var items []string
_, err = pgdb.sql.Query(&items, entity.query)
if err != nil {
t.Error("get the list of "+entity.name+"return an error: ", err)
}
for _, item := range items {
_, err = pgdb.sql.Exec("drop " + entity.name + " " + item + " cascade")
if err != nil {
t.Error("drop ", entity.name, " ", item, " return an error: ", err)
}
}
}
err = db.Close()
if err != nil {
t.Error("db.Close() return an error: ", err)
}
}
return db, cleanFn
}
func TestInit(t *testing.T) { func TestInit(t *testing.T) {
db := Init(test_host, test_coll) _, dbclose := testDbInit(t)
defer db.Close() defer dbclose()
}
func del(db DB) {
db.Close()
session, _ := mgo.Dial(test_host)
defer session.Close()
session.DB(test_coll).DropDatabase()
} }

241
lib/database/mgo.go
View file

@ -1,241 +0,0 @@
package database
import (
"errors"
mgo "gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
const (
visited_coll = "visited"
downloaded_coll = "downloaded"
tags_coll = "tags"
)
type mgoDB struct {
session *mgo.Session
name string
}
func (db *mgoDB) initIndexes() {
dbCopy := db.session.Copy()
booksColl := dbCopy.DB(db.name).C(books_coll)
go indexBooks(booksColl)
statsColl := dbCopy.DB(db.name).C(stats_coll)
go indexStats(statsColl)
newsColl := dbCopy.DB(db.name).C(news_coll)
go indexNews(newsColl)
}
func (db *mgoDB) Close() {
db.session.Close()
}
func (db *mgoDB) Copy() DB {
dbCopy := new(mgoDB)
dbCopy.session = db.session.Copy()
dbCopy.name = db.name
return dbCopy
}
func (db *mgoDB) AddBook(book map[string]interface{}) error {
booksColl := db.session.DB(db.name).C(books_coll)
return addBook(booksColl, book)
}
func (db *mgoDB) GetBooks(query string, length int, start int) (books []Book, num int, err error) {
booksColl := db.session.DB(db.name).C(books_coll)
return getBooks(booksColl, query, length, start)
}
func (db *mgoDB) GetBooksIter() Iter {
booksColl := db.session.DB(db.name).C(books_coll)
return getBooksIter(booksColl)
}
func (db *mgoDB) GetNewBooks(query string, length int, start int) (books []Book, num int, err error) {
booksColl := db.session.DB(db.name).C(books_coll)
return getNewBooks(booksColl, query, length, start)
}
func (db *mgoDB) GetBookId(id string) (Book, error) {
booksColl := db.session.DB(db.name).C(books_coll)
return getBookId(booksColl, id)
}
func (db *mgoDB) DeleteBook(id string) error {
booksColl := db.session.DB(db.name).C(books_coll)
return deleteBook(booksColl, id)
}
func (db *mgoDB) UpdateBook(id string, data map[string]interface{}) error {
booksColl := db.session.DB(db.name).C(books_coll)
return updateBook(booksColl, id, data)
}
func (db *mgoDB) FlagBadQuality(id string, user string) error {
booksColl := db.session.DB(db.name).C(books_coll)
return flagBadQuality(booksColl, id, user)
}
func (db *mgoDB) ActiveBook(id string) error {
booksColl := db.session.DB(db.name).C(books_coll)
return activeBook(booksColl, id)
}
func (db *mgoDB) IsBookActive(id string) bool {
booksColl := db.session.DB(db.name).C(books_coll)
return isBookActive(booksColl, id)
}
func (db *mgoDB) User(name string) *User {
userColl := db.session.DB(db.name).C(user_coll)
return getUser(userColl, name)
}
func (db *mgoDB) AddUser(name string, pass string) error {
userColl := db.session.DB(db.name).C(user_coll)
return addUser(userColl, name, pass)
}
func (db *mgoDB) AddNews(text string) error {
newsColl := db.session.DB(db.name).C(news_coll)
return addNews(newsColl, text)
}
func (db *mgoDB) GetNews(num int, days int) (news []News, err error) {
newsColl := db.session.DB(db.name).C(news_coll)
return getNews(newsColl, num, days)
}
// TODO: split code in files
func (db *mgoDB) AddStats(stats interface{}) error {
statsColl := db.session.DB(db.name).C(stats_coll)
return statsColl.Insert(stats)
}
/* Get the most visited books
*/
func (db *mgoDB) GetVisitedBooks() (books []Book, err error) {
visitedColl := db.session.DB(db.name).C(visited_coll)
bookId, err := GetBooksVisited(visitedColl)
if err != nil {
return nil, err
}
books = make([]Book, len(bookId))
for i, id := range bookId {
booksColl := db.session.DB(db.name).C(books_coll)
booksColl.Find(bson.M{"_id": id}).One(&books[i])
books[i].Id = bson.ObjectId(books[i].Id).Hex()
}
return
}
func (db *mgoDB) UpdateMostVisited() error {
var u dbUpdate
u.src = db.session.DB(db.name).C(stats_coll)
u.dst = db.session.DB(db.name).C(visited_coll)
return u.UpdateMostBooks("book")
}
/* Get the most downloaded books
*/
func (db *mgoDB) GetDownloadedBooks() (books []Book, err error) {
downloadedColl := db.session.DB(db.name).C(downloaded_coll)
bookId, err := GetBooksVisited(downloadedColl)
if err != nil {
return nil, err
}
books = make([]Book, len(bookId))
for i, id := range bookId {
booksColl := db.session.DB(db.name).C(books_coll)
booksColl.Find(bson.M{"_id": id}).One(&books[i])
books[i].Id = bson.ObjectId(books[i].Id).Hex()
}
return
}
func (db *mgoDB) UpdateDownloadedBooks() error {
var u dbUpdate
u.src = db.session.DB(db.name).C(stats_coll)
u.dst = db.session.DB(db.name).C(downloaded_coll)
return u.UpdateMostBooks("download")
}
func (db *mgoDB) GetTags() ([]string, error) {
tagsColl := db.session.DB(db.name).C(tags_coll)
return GetTags(tagsColl)
}
func (db *mgoDB) UpdateTags() error {
var u dbUpdate
u.src = db.session.DB(db.name).C(books_coll)
u.dst = db.session.DB(db.name).C(tags_coll)
return u.UpdateTags()
}
func (db *mgoDB) GetVisits(visitType VisitType) ([]Visits, error) {
var coll *mgo.Collection
switch visitType {
case Hourly_visits:
coll = db.session.DB(db.name).C(hourly_visits_coll)
case Daily_visits:
coll = db.session.DB(db.name).C(daily_visits_coll)
case Monthly_visits:
coll = db.session.DB(db.name).C(monthly_visits_coll)
case Hourly_downloads:
coll = db.session.DB(db.name).C(hourly_downloads_coll)
case Daily_downloads:
coll = db.session.DB(db.name).C(daily_downloads_coll)
case Monthly_downloads:
coll = db.session.DB(db.name).C(monthly_downloads_coll)
default:
return nil, errors.New("Not valid VisitType")
}
return GetVisits(coll)
}
func (db *mgoDB) UpdateHourVisits() error {
var u dbUpdate
u.src = db.session.DB(db.name).C(stats_coll)
u.dst = db.session.DB(db.name).C(hourly_visits_coll)
return u.UpdateHourVisits(false)
}
func (db *mgoDB) UpdateDayVisits() error {
var u dbUpdate
u.src = db.session.DB(db.name).C(stats_coll)
u.dst = db.session.DB(db.name).C(daily_visits_coll)
return u.UpdateDayVisits(false)
}
func (db *mgoDB) UpdateMonthVisits() error {
var u dbUpdate
u.src = db.session.DB(db.name).C(stats_coll)
u.dst = db.session.DB(db.name).C(monthly_visits_coll)
return u.UpdateMonthVisits(false)
}
func (db *mgoDB) UpdateHourDownloads() error {
var u dbUpdate
u.src = db.session.DB(db.name).C(stats_coll)
u.dst = db.session.DB(db.name).C(hourly_downloads_coll)
return u.UpdateHourVisits(true)
}
func (db *mgoDB) UpdateDayDownloads() error {
var u dbUpdate
u.src = db.session.DB(db.name).C(stats_coll)
u.dst = db.session.DB(db.name).C(daily_downloads_coll)
return u.UpdateDayVisits(true)
}
func (db *mgoDB) UpdateMonthDownloads() error {
var u dbUpdate
u.src = db.session.DB(db.name).C(stats_coll)
u.dst = db.session.DB(db.name).C(monthly_downloads_coll)
return u.UpdateMonthVisits(true)
}

53
lib/database/news.go
View file

@ -1,49 +1,30 @@
package database package database
import ( import (
log "github.com/cihub/seelog"
"time" "time"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
) )
const ( // New entry in the news table
news_coll = "news" type New struct {
) ID int
type News struct {
Date time.Time Date time.Time
Text string Text string
} }
func indexNews(coll *mgo.Collection) { // AddNews creates a new entry
idx := mgo.Index{ func (db *pgDB) AddNews(text string) error {
Key: []string{"-date"}, return db.sql.Create(&New{
Background: true, Text: text,
} Date: time.Now(),
err := coll.EnsureIndex(idx) })
if err != nil {
log.Error("Error indexing news: ", err)
}
} }
func addNews(coll *mgo.Collection, text string) error { // GetNews returns all the news for the last 'days' limiting with a maximum of 'num' results
var news News func (db *pgDB) GetNews(num int, days int) ([]New, error) {
news.Text = text var news []New
news.Date = time.Now() err := db.sql.Model(&news).
return coll.Insert(news) Limit(num).
} Order("date DESC").
Select()
func getNews(coll *mgo.Collection, num int, days int) (news []News, err error) { return news, err
query := bson.M{}
if days != 0 {
duration := time.Duration(-24*days) * time.Hour
date := time.Now().Add(duration)
query = bson.M{"date": bson.M{"$gt": date}}
}
q := coll.Find(query).Sort("-date").Limit(num)
err = q.All(&news)
return
} }

30
lib/database/news_test.go
View file

@ -5,8 +5,8 @@ import "testing"
func TestNews(t *testing.T) { func TestNews(t *testing.T) {
const text = "Some news text" const text = "Some news text"
db := Init(test_host, test_coll) db, dbclose := testDbInit(t)
defer del(db) defer dbclose()
err := db.AddNews(text) err := db.AddNews(text)
if err != nil { if err != nil {
@ -24,3 +24,29 @@ func TestNews(t *testing.T) {
t.Errorf("News text don't match : '", news[0].Text, "' <=> '", text, "'") t.Errorf("News text don't match : '", news[0].Text, "' <=> '", text, "'")
} }
} }
func TestTwoNews(t *testing.T) {
const text = "Some news text"
const text2 = "More news"
db, dbclose := testDbInit(t)
defer dbclose()
err := db.AddNews(text)
if err != nil {
t.Errorf("db.News(", text, ") return an error: ", err)
}
err = db.AddNews(text2)
if err != nil {
t.Errorf("db.News(", text, ") return an error: ", err)
}
news, err := db.GetNews(2, 1)
if err != nil {
t.Fatalf("db.GetNews() return an error: ", err)
}
if len(news) < 2 {
t.Fatalf("No news found.")
}
}

30
lib/database/ro.go
View file

@ -8,15 +8,11 @@ type roDB struct {
db DB db DB
} }
func (db *roDB) Close() { func (db *roDB) Close() error {
db.db.Close() return db.db.Close()
} }
func (db *roDB) Copy() DB { func (db *roDB) AddBook(book Book) error {
return &roDB{db.db.Copy()}
}
func (db *roDB) AddBook(book map[string]interface{}) error {
return errors.New("RO database") return errors.New("RO database")
} }
@ -24,10 +20,6 @@ func (db *roDB) GetBooks(query string, length int, start int) (books []Book, num
return db.db.GetBooks(query, length, start) return db.db.GetBooks(query, length, start)
} }
func (db *roDB) GetBooksIter() Iter {
return db.db.GetBooksIter()
}
func (db *roDB) GetNewBooks(query string, length int, start int) (books []Book, num int, err error) { func (db *roDB) GetNewBooks(query string, length int, start int) (books []Book, num int, err error) {
return db.db.GetNewBooks(query, length, start) return db.db.GetNewBooks(query, length, start)
} }
@ -56,11 +48,19 @@ func (db *roDB) IsBookActive(id string) bool {
return db.db.IsBookActive(id) return db.db.IsBookActive(id)
} }
func (db *roDB) User(name string) *User { func (db *roDB) AddUser(name string, pass string) error {
return db.db.User(name) return errors.New("RO database")
} }
func (db *roDB) AddUser(name string, pass string) error { func (db *roDB) GetRole(name string) (string, error) {
return db.db.GetRole(name)
}
func (db *roDB) ValidPassword(name string, pass string) bool {
return db.db.ValidPassword(name, pass)
}
func (db *roDB) SetPassword(name string, pass string) error {
return errors.New("RO database") return errors.New("RO database")
} }
@ -68,7 +68,7 @@ func (db *roDB) AddNews(text string) error {
return errors.New("RO database") return errors.New("RO database")
} }
func (db *roDB) GetNews(num int, days int) (news []News, err error) { func (db *roDB) GetNews(num int, days int) (news []New, err error) {
return db.db.GetNews(num, days) return db.db.GetNews(num, days)
} }

240
lib/database/stats.go
View file

@ -1,33 +1,10 @@
// TODO
package database package database
import ( import (
log "github.com/cihub/seelog"
"time" "time"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
) )
const (
stats_coll = "statistics"
hourly_visits_coll = "visits.hourly"
daily_visits_coll = "visits.daily"
monthly_visits_coll = "visits.monthly"
hourly_downloads_coll = "downloads.hourly"
daily_downloads_coll = "downloads.daily"
monthly_downloads_coll = "downloads.monthly"
// FIXME: this should return to the config.go
TAGS_DISPLAY = 50
BOOKS_FRONT_PAGE = 6
)
type dbUpdate struct {
src *mgo.Collection
dst *mgo.Collection
}
type VisitType int type VisitType int
const ( const (
@ -44,210 +21,63 @@ type Visits struct {
Count int "count" Count int "count"
} }
func indexStats(coll *mgo.Collection) { // TODO: split code in files
indexes := []mgo.Index{ func (db *pgDB) AddStats(stats interface{}) error {
{
Key: []string{"section"},
Background: true,
},
{
Key: []string{"-date", "section"},
Background: true,
},
}
for _, idx := range indexes {
err := coll.EnsureIndex(idx)
if err != nil {
log.Error("Error indexing stats: ", err)
}
}
}
func GetTags(tagsColl *mgo.Collection) ([]string, error) {
var result []struct {
Tag string "_id"
}
err := tagsColl.Find(nil).Sort("-count").All(&result)
if err != nil {
return nil, err
}
tags := make([]string, len(result))
for i, r := range result {
tags[i] = r.Tag
}
return tags, nil
}
func GetBooksVisited(visitedColl *mgo.Collection) ([]bson.ObjectId, error) {
var result []struct {
Book bson.ObjectId "_id"
}
err := visitedColl.Find(nil).Sort("-count").All(&result)
if err != nil {
return nil, err
}
books := make([]bson.ObjectId, len(result))
for i, r := range result {
books[i] = r.Book
}
return books, nil
}
func GetVisits(visitsColl *mgo.Collection) ([]Visits, error) {
var result []Visits
err := visitsColl.Find(nil).All(&result)
return result, err
}
func (u *dbUpdate) UpdateTags() error {
var tags []struct {
Tag string "_id"
Count int "count"
}
err := u.src.Pipe([]bson.M{
{"$project": bson.M{"subject": 1}},
{"$unwind": "$subject"},
{"$group": bson.M{"_id": "$subject", "count": bson.M{"$sum": 1}}},
{"$sort": bson.M{"count": -1}},
{"$limit": TAGS_DISPLAY},
}).All(&tags)
if err != nil {
return err
}
u.dst.DropCollection()
for _, tag := range tags {
err = u.dst.Insert(tag)
if err != nil {
return err
}
}
return nil return nil
} }
func (u *dbUpdate) UpdateMostBooks(section string) error { /* Get the most visited books
const numDays = 30 */
start := time.Now().UTC().Add(-numDays * 24 * time.Hour) func (db *pgDB) GetVisitedBooks() (books []Book, err error) {
return []Book{}, nil
}
var books []struct { func (db *pgDB) UpdateMostVisited() error {
Book string "_id"
Count int "count"
}
err := u.src.Pipe([]bson.M{
{"$match": bson.M{"date": bson.M{"$gt": start}, "section": section}},
{"$project": bson.M{"id": 1}},
{"$group": bson.M{"_id": "$id", "count": bson.M{"$sum": 1}}},
{"$sort": bson.M{"count": -1}},
{"$limit": BOOKS_FRONT_PAGE},
}).All(&books)
if err != nil {
return err
}
u.dst.DropCollection()
for _, book := range books {
err = u.dst.Insert(book)
if err != nil {
return err
}
}
return nil return nil
} }
func (u *dbUpdate) UpdateHourVisits(isDownloads bool) error { /* Get the most downloaded books
const numDays = 2 */
spanStore := numDays * 24 * time.Hour func (db *pgDB) GetDownloadedBooks() (books []Book, err error) {
return u.updateVisits(hourInc, spanStore, isDownloads) return []Book{}, nil
} }
func (u *dbUpdate) UpdateDayVisits(isDownloads bool) error { func (db *pgDB) UpdateDownloadedBooks() error {
const numDays = 30 return nil
spanStore := numDays * 24 * time.Hour
return u.updateVisits(dayInc, spanStore, isDownloads)
} }
func (u *dbUpdate) UpdateMonthVisits(isDownloads bool) error { func (db *pgDB) GetTags() ([]string, error) {
const numDays = 365 return []string{}, nil
spanStore := numDays * 24 * time.Hour
return u.updateVisits(monthInc, spanStore, isDownloads)
} }
func hourInc(date time.Time) time.Time { func (db *pgDB) UpdateTags() error {
const span = time.Hour return nil
return date.Add(span).Truncate(span)
} }
func dayInc(date time.Time) time.Time { func (db *pgDB) GetVisits(visitType VisitType) ([]Visits, error) {
const span = 24 * time.Hour return []Visits{}, nil
return date.Add(span).Truncate(span)
} }
func monthInc(date time.Time) time.Time { func (db *pgDB) UpdateHourVisits() error {
const span = 24 * time.Hour return nil
return date.AddDate(0, 1, 1-date.Day()).Truncate(span)
} }
func (u *dbUpdate) updateVisits(incTime func(time.Time) time.Time, spanStore time.Duration, isDownloads bool) error { func (db *pgDB) UpdateDayVisits() error {
start := u.calculateStart(spanStore) return nil
for start.Before(time.Now().UTC()) {
stop := incTime(start)
var count int
var err error
if isDownloads {
count, err = u.countDownloads(start, stop)
} else {
count = u.countVisits(start, stop)
}
if err != nil {
return err
}
err = u.dst.Insert(bson.M{"date": start, "count": count})
if err != nil {
return err
}
start = stop
}
_, err := u.dst.RemoveAll(bson.M{"date": bson.M{"$lt": time.Now().UTC().Add(-spanStore)}})
return err
} }
func (u *dbUpdate) calculateStart(spanStore time.Duration) time.Time { func (db *pgDB) UpdateMonthVisits() error {
var date struct { return nil
Id bson.ObjectId `bson:"_id"`
Date time.Time `bson:"date"`
}
err := u.dst.Find(bson.M{}).Sort("-date").One(&date)
if err == nil {
u.dst.RemoveId(date.Id)
return date.Date
}
return time.Now().UTC().Add(-spanStore).Truncate(time.Hour)
} }
func (u *dbUpdate) countVisits(start time.Time, stop time.Time) int { func (db *pgDB) UpdateHourDownloads() error {
var result struct { return nil
Count int "count"
}
err := u.src.Pipe([]bson.M{
{"$match": bson.M{"date": bson.M{"$gte": start, "$lt": stop}}},
{"$group": bson.M{"_id": "$session"}},
{"$group": bson.M{"_id": 1, "count": bson.M{"$sum": 1}}},
}).One(&result)
if err != nil {
return 0
}
return result.Count
} }
func (u *dbUpdate) countDownloads(start time.Time, stop time.Time) (int, error) { func (db *pgDB) UpdateDayDownloads() error {
query := bson.M{"date": bson.M{"$gte": start, "$lt": stop}, "section": "download"} return nil
return u.src.Find(query).Count() }
func (db *pgDB) UpdateMonthDownloads() error {
return nil
} }

108
lib/database/users.go
View file

@ -8,50 +8,21 @@ import (
"errors" "errors"
"golang.org/x/crypto/scrypt" "golang.org/x/crypto/scrypt"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
) )
const ( type user struct {
user_coll = "users" ID int
pass_salt = "ImperialLibSalt" Username string
) Password []byte
Salt []byte
type User struct { Role string
user db_user
err error
coll *mgo.Collection
} }
type db_user struct { func (db *pgDB) AddUser(name string, pass string) error {
User string
Pass []byte
Salt []byte
Role string
}
func getUser(coll *mgo.Collection, name string) *User {
u := new(User)
if !validUserName(name) {
u.err = errors.New("Invalid username")
return u
}
u.coll = coll
err := u.coll.Find(bson.M{"user": name}).One(&u.user)
if err != nil {
log.Warn("Error on database checking user ", name, ": ", err)
u.err = errors.New("User not found")
return u
}
return u
}
func addUser(coll *mgo.Collection, name string, pass string) error {
if !validUserName(name) { if !validUserName(name) {
return errors.New("Invalid user name") return errors.New("Invalid user name")
} }
num, err := coll.Find(bson.M{"user": name}).Count() num, err := db.sql.Model(&user{}).Where("username = ?", name).Count()
if err != nil { if err != nil {
log.Error("Error on database checking user ", name, ": ", err) log.Error("Error on database checking user ", name, ": ", err)
return errors.New("An error happen on the database") return errors.New("An error happen on the database")
@ -60,41 +31,54 @@ func addUser(coll *mgo.Collection, name string, pass string) error {
return errors.New("User name already exist") return errors.New("User name already exist")
} }
var user db_user hpass, salt, err := hashPass(pass)
user.Pass, user.Salt, err = hashPass(pass)
if err != nil { if err != nil {
log.Error("Error hashing password: ", err) log.Error("Error hashing password: ", err)
return errors.New("An error happen storing the password") return errors.New("An error happen storing the password")
} }
user.User = name u := user{
user.Role = "" Username: name,
return coll.Insert(user) Password: hpass,
Salt: salt,
Role: "",
}
return db.sql.Create(&u)
} }
func validUserName(name string) bool { func (db *pgDB) GetRole(name string) (string, error) {
return name != "" var u user
err := db.sql.Model(&u).Where("username = ?", name).Select()
return u.Role, err
} }
func (u User) Valid(pass string) bool { func (db *pgDB) ValidPassword(name string, pass string) bool {
if u.err != nil { var u user
err := db.sql.Model(&u).Where("username = ?", name).Select()
if err != nil {
return false return false
} }
return validatePass(pass, u.user)
}
func (u User) Role() string { hash, err := calculateHash(pass, u.Salt)
return u.user.Role if err != nil {
} return false
func (u *User) SetPassword(pass string) error {
if u.err != nil {
return u.err
} }
return bytes.Compare(u.Password, hash) == 0
}
func (db *pgDB) SetPassword(name string, pass string) error {
hash, salt, err := hashPass(pass) hash, salt, err := hashPass(pass)
if err != nil { if err != nil {
return err return err
} }
return u.coll.Update(bson.M{"user": u.user.User}, bson.M{"$set": bson.M{"pass": hash, "salt": salt}}) _, err = db.sql.Model(user{}).
Set("pass = ?, salt = ?", hash, salt).
Where("username = ?", name).
Update()
return err
}
func validUserName(name string) bool {
return name != ""
} }
func hashPass(pass string) (hash []byte, salt []byte, err error) { func hashPass(pass string) (hash []byte, salt []byte, err error) {
@ -107,23 +91,13 @@ func hashPass(pass string) (hash []byte, salt []byte, err error) {
} }
func genSalt() ([]byte, error) { func genSalt() ([]byte, error) {
const ( const saltLen = 64
saltLen = 64
)
b := make([]byte, saltLen) b := make([]byte, saltLen)
_, err := rand.Read(b) _, err := rand.Read(b)
return b, err return b, err
} }
func validatePass(pass string, user db_user) bool {
hash, err := calculateHash(pass, user.Salt)
if err != nil {
return false
}
return bytes.Compare(user.Pass, hash) == 0
}
func calculateHash(pass string, salt []byte) ([]byte, error) { func calculateHash(pass string, salt []byte) ([]byte, error) {
const ( const (
N = 16384 N = 16384

32
lib/database/users_test.go
View file

@ -7,37 +7,37 @@ const (
) )
func TestUserEmpty(t *testing.T) { func TestUserEmpty(t *testing.T) {
db := Init(test_host, test_coll) db, dbclose := testDbInit(t)
defer del(db) defer dbclose()
if db.User("").Valid("") { if db.ValidPassword("", "") {
t.Errorf("user.Valid() with an empty password return true") t.Errorf("ValidPassword() with an empty password return true")
} }
} }
func TestAddUser(t *testing.T) { func TestAddUser(t *testing.T) {
db := Init(test_host, test_coll) db, dbclose := testDbInit(t)
defer del(db) defer dbclose()
tAddUser(t, db) testAddUser(t, db)
if !db.User(name).Valid(pass) { if !db.ValidPassword(name, pass) {
t.Errorf("user.Valid() return false for a valid user") t.Errorf("ValidPassword() return false for a valid user")
} }
} }
func TestEmptyUsername(t *testing.T) { func TestEmptyUsername(t *testing.T) {
db := Init(test_host, test_coll) db, dbclose := testDbInit(t)
defer del(db) defer dbclose()
tAddUser(t, db) testAddUser(t, db)
if db.User("").Valid(pass) { if db.ValidPassword("", pass) {
t.Errorf("user.Valid() return true for an invalid user") t.Errorf("ValidPassword() return true for an invalid user")
} }
} }
func tAddUser(t *testing.T, db DB) { func testAddUser(t *testing.T, db DB) {
err := db.AddUser(name, pass) err := db.AddUser(name, pass)
if err != nil { if err != nil {
t.Errorf("db.Adduser(", name, ", ", pass, ") return an error: ", err) t.Errorf("db.Adduser(%v, %v) return an error: %v", name, pass, err)
} }
} }

38
lib/parser/language.go
View file

@ -3,23 +3,24 @@ package parser
import ( import (
"io/ioutil" "io/ioutil"
"strings" "strings"
"unicode/utf8"
"github.com/jmhodges/gocld2" "github.com/jmhodges/gocld2"
"github.com/meskio/epubgo" "github.com/meskio/epubgo"
) )
func GuessLang(epub *epubgo.Epub, orig_langs []string) []string { func GuessLang(epub *epubgo.Epub, origLangs []string) string {
spine, err := epub.Spine() spine, err := epub.Spine()
if err != nil { if err != nil {
return orig_langs return normalizeLangs(origLangs)
} }
var err_spine error var errSpine error
err_spine = nil errSpine = nil
langs := []string{} langs := []string{}
for err_spine == nil { for errSpine == nil {
html, err := spine.Open() html, err := spine.Open()
err_spine = spine.Next() errSpine = spine.Next()
if err != nil { if err != nil {
continue continue
} }
@ -29,14 +30,16 @@ func GuessLang(epub *epubgo.Epub, orig_langs []string) []string {
if err != nil { if err != nil {
continue continue
} }
langs = append(langs, cld2.Detect(string(buff))) if utf8.Valid(buff) {
langs = append(langs, cld2.Detect(string(buff)))
}
} }
lang := commonLang(langs) lang := commonLang(langs)
if lang != "un" && differentLang(lang, orig_langs) { if lang == "un" {
return []string{lang} return normalizeLangs(origLangs)
} }
return orig_langs return lang
} }
func commonLang(langs []string) string { func commonLang(langs []string) string {
@ -56,11 +59,14 @@ func commonLang(langs []string) string {
return lang return lang
} }
func differentLang(lang string, orig_langs []string) bool { func normalizeLangs(langs []string) string {
orig_lang := "un" lang := "un"
if len(orig_langs) > 0 && len(orig_langs) >= 2 { if len(langs) > 0 {
orig_lang = strings.ToLower(orig_langs[0][0:2]) lang = langs[0]
if len(lang) > 3 {
lang = lang[0:2]
}
lang = strings.ToLower(lang)
} }
return "un"
return orig_lang != lang
} }

49
lib/parser/parser.go
View file

@ -5,45 +5,46 @@ import (
"strings" "strings"
"github.com/meskio/epubgo" "github.com/meskio/epubgo"
"gitlab.com/trantor/trantor/lib/database"
) )
type MetaData map[string]interface{} func EpubMetadata(epub *epubgo.Epub) database.Book {
book := database.Book{}
func EpubMetadata(epub *epubgo.Epub) MetaData {
metadata := MetaData{}
for _, m := range epub.MetadataFields() { for _, m := range epub.MetadataFields() {
data, err := epub.Metadata(m) data, err := epub.Metadata(m)
if err != nil { if err != nil {
continue continue
} }
switch m { switch m {
case "title":
book.Title = cleanStr(strings.Join(data, ", "))
case "creator": case "creator":
metadata["author"] = parseAuthr(data) book.Author = parseAuthr(data)
case "contributor":
book.Contributor = cleanStr(strings.Join(data, ", "))
case "publisher":
book.Publisher = cleanStr(strings.Join(data, ", "))
case "description": case "description":
metadata[m] = parseDescription(data) book.Description = parseDescription(data)
case "subject": case "subject":
metadata[m] = parseSubject(data) book.Subject = parseSubject(data)
case "date": case "date":
metadata[m] = parseDate(data) book.Date = parseDate(data)
case "language": case "language":
metadata["lang"] = GuessLang(epub, data) book.Lang = GuessLang(epub, data)
case "title", "contributor", "publisher":
metadata[m] = cleanStr(strings.Join(data, ", "))
case "identifier": case "identifier":
attr, _ := epub.MetadataAttr(m) attr, _ := epub.MetadataAttr(m)
for i, d := range data { for i, d := range data {
if attr[i]["scheme"] == "ISBN" { if attr[i]["scheme"] == "ISBN" {
isbn := ISBN(d) isbn := ISBN(d)
if isbn != "" { if isbn != "" {
metadata["isbn"] = isbn book.Isbn = isbn
} }
} }
} }
default:
metadata[m] = strings.Join(data, ", ")
} }
} }
return metadata return book
} }
func cleanStr(str string) string { func cleanStr(str string) string {
@ -88,9 +89,21 @@ func parseDescription(description []string) string {
} }
func parseSubject(subject []string) []string { func parseSubject(subject []string) []string {
var res []string parsed := subject
for _, s := range subject { for _, sep := range []string{"/", ","} {
res = append(res, strings.Split(s, " / ")...) p2 := []string{}
for _, s := range subject {
p2 = append(p2, strings.Split(s, sep)...)
}
parsed = p2
}
res := []string{}
for _, s := range parsed {
sub := strings.Trim(s, " ")
sub = strings.ToLower(sub)
if len(sub) != 0 {
res = append(res, sub)
}
} }
return res return res
} }

2
lib/session.go
View file

@ -29,7 +29,7 @@ func GetSession(r *http.Request, db database.DB) (s *Session) {
s.S, err = sesStore.Get(r, "session") s.S, err = sesStore.Get(r, "session")
if err == nil && !s.S.IsNew { if err == nil && !s.S.IsNew {
s.User, _ = s.S.Values["user"].(string) s.User, _ = s.S.Values["user"].(string)
s.Role = db.User(s.User).Role() s.Role, _ = db.GetRole(s.User)
} }
if s.S.IsNew { if s.S.IsNew {

11
lib/stats.go
View file

@ -56,18 +56,16 @@ func (sg StatsGatherer) Gather(function func(handler)) func(http.ResponseWriter,
return func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) {
log.Info("Query ", r.Method, " ", r.RequestURI) log.Info("Query ", r.Method, " ", r.RequestURI)
db := sg.db.Copy()
h := handler{ h := handler{
store: sg.store, store: sg.store,
db: db, db: sg.db,
template: sg.template, template: sg.template,
hostname: sg.hostname, hostname: sg.hostname,
w: w, w: w,
r: r, r: r,
sess: GetSession(r, db), sess: GetSession(r, sg.db),
ro: sg.ro, ro: sg.ro,
} }
defer h.db.Close()
function(h) function(h)
sg.channel <- statsRequest{time.Now(), mux.Vars(r), h.sess, r} sg.channel <- statsRequest{time.Now(), mux.Vars(r), h.sess, r}
@ -82,9 +80,6 @@ type statsRequest struct {
} }
func (sg StatsGatherer) worker() { func (sg StatsGatherer) worker() {
db := sg.db.Copy()
defer db.Close()
for req := range sg.channel { for req := range sg.channel {
stats := make(map[string]interface{}) stats := make(map[string]interface{})
appendFiles(req.r, stats) appendFiles(req.r, stats)
@ -94,7 +89,7 @@ func (sg StatsGatherer) worker() {
stats["version"] = stats_version stats["version"] = stats_version
stats["method"] = req.r.Method stats["method"] = req.r.Method
stats["date"] = req.date stats["date"] = req.date
db.AddStats(stats) sg.db.AddStats(stats)
} }
} }

24
lib/upload.go
View file

@ -32,11 +32,8 @@ type uploadRequest struct {
} }
func uploadWorker(database database.DB, store storage.Store) { func uploadWorker(database database.DB, store storage.Store) {
db := database.Copy()
defer db.Close()
for req := range uploadChannel { for req := range uploadChannel {
processFile(req, db, store) processFile(req, database, store)
} }
} }
@ -50,22 +47,23 @@ func processFile(req uploadRequest, db database.DB, store storage.Store) {
} }
defer epub.Close() defer epub.Close()
id := genId() book := parser.EpubMetadata(epub)
metadata := parser.EpubMetadata(epub) book.Id = genId()
metadata["id"] = id
metadata["cover"] = GetCover(epub, id, store)
req.file.Seek(0, 0) req.file.Seek(0, 0)
size, err := store.Store(id, req.file, epubFile) size, err := store.Store(book.Id, req.file, epubFile)
if err != nil { if err != nil {
log.Error("Error storing book (", id, "): ", err) log.Error("Error storing book (", book.Id, "): ", err)
return return
} }
metadata["filesize"] = size book.FileSize = int(size)
err = db.AddBook(metadata) book.Cover = GetCover(epub, book.Id, store)
err = db.AddBook(book)
log.Error(":", book.Lang, ":")
if err != nil { if err != nil {
log.Error("Error storing metadata (", id, "): ", err) log.Error("Error storing metadata (", book.Id, "): ", err)
return return
} }
log.Info("File uploaded: ", req.filename) log.Info("File uploaded: ", req.filename)

6
lib/user.go
View file

@ -21,7 +21,7 @@ func loginHandler(h handler) {
func loginPostHandler(h handler) { func loginPostHandler(h handler) {
user := h.r.FormValue("user") user := h.r.FormValue("user")
pass := h.r.FormValue("pass") pass := h.r.FormValue("pass")
if h.db.User(user).Valid(pass) { if h.db.ValidPassword(user, pass) {
log.Info("User ", user, " log in") log.Info("User ", user, " log in")
h.sess.LogIn(user) h.sess.LogIn(user)
h.sess.Notify("Successful login!", "Welcome "+user, "success") h.sess.Notify("Successful login!", "Welcome "+user, "success")
@ -74,12 +74,12 @@ func settingsHandler(h handler) {
pass1 := h.r.FormValue("password1") pass1 := h.r.FormValue("password1")
pass2 := h.r.FormValue("password2") pass2 := h.r.FormValue("password2")
switch { switch {
case !h.db.User(h.sess.User).Valid(current_pass): case !h.db.ValidPassword(h.sess.User, current_pass):
h.sess.Notify("Password error!", "The current password given don't match with the user password. Try again", "error") h.sess.Notify("Password error!", "The current password given don't match with the user password. Try again", "error")
case pass1 != pass2: case pass1 != pass2:
h.sess.Notify("Passwords don't match!", "The new password and the confirmation password don't match. Try again", "error") h.sess.Notify("Passwords don't match!", "The new password and the confirmation password don't match. Try again", "error")
default: default:
h.db.User(h.sess.User).SetPassword(pass1) h.db.SetPassword(h.sess.User, pass1)
h.sess.Notify("Password updated!", "Your new password is correctly set.", "success") h.sess.Notify("Password updated!", "Your new password is correctly set.", "success")
} }
h.sess.Save(h.w, h.r) h.sess.Save(h.w, h.r)

15
main.go
View file

@ -15,7 +15,9 @@ import (
func main() { func main() {
var ( var (
httpAddr = flag.String("addr", ":8080", "HTTP service address") httpAddr = flag.String("addr", ":8080", "HTTP service address")
dbIP = flag.String("db-ip", "127.0.0.1", "IP address of the database") dbAddr = flag.String("db-addr", "localhost:5432", "IP address of the database")
dbUser = flag.String("db-user", "", "User name to access the database")
dbPassword = flag.String("db-password", "", "Password to access the database")
dbName = flag.String("db-name", "trantor", "Name of the database") dbName = flag.String("db-name", "trantor", "Name of the database")
storePath = flag.String("store", "store", "Path of the books storage") storePath = flag.String("store", "store", "Path of the books storage")
assetsPath = flag.String("assets", ".", "Path of the assets (templates, css, js, img)") assetsPath = flag.String("assets", ".", "Path of the assets (templates, css, js, img)")
@ -32,7 +34,16 @@ func main() {
} }
log.Info("Start the imperial library of trantor") log.Info("Start the imperial library of trantor")
db := database.Init(*dbIP, *dbName) db, err := database.Init(database.Options{
Addr: *dbAddr,
User: *dbUser,
Password: *dbPassword,
Name: *dbName,
})
if err != nil {
log.Critical("Problem initializing database: ", err)
os.Exit(1)
}
defer db.Close() defer db.Close()
store, err := storage.Init(*storePath) store, err := storage.Init(*storePath)

2
templates/book.html
View file

@ -40,7 +40,7 @@ function delBook(){
{{if .Subject}}<dt>Tags</dt> <dd>{{range .Subject}}<a href="/search/?q=subject:{{.}}">{{.}}</a>, {{end}}</dd>{{end}} {{if .Subject}}<dt>Tags</dt> <dd>{{range .Subject}}<a href="/search/?q=subject:{{.}}">{{.}}</a>, {{end}}</dd>{{end}}
{{if .Isbn}}<dt>ISBN</dt> <dd>{{.Isbn}}</dd>{{end}} {{if .Isbn}}<dt>ISBN</dt> <dd>{{.Isbn}}</dd>{{end}}
{{if .Date}}<dt>Date</dt> <dd>{{.Date}}</dd>{{end}} {{if .Date}}<dt>Date</dt> <dd>{{.Date}}</dd>{{end}}
{{if .Lang}}<dt>Lang</dt> <dd>{{range .Lang}}<a href="/search/?q=lang:{{.}}">{{.}}</a> {{end}}</dd>{{end}} {{if .Lang}}<dt>Lang</dt> <dd><a href="/search/?q=lang:{{.Lang}}">{{.Lang}}</a> </dd>{{end}}
</dl> </dl>
</div> </div>
<div class="span3"> <div class="span3">

7
templates/edit.html
View file

@ -53,12 +53,9 @@
</div> </div>
</div> </div>
<div class="control-group"> <div class="control-group">
<label class="control-label" for="langs">Langs</label> <label class="control-label" for="lang">Lang</label>
<div class="controls"> <div class="controls">
{{range .Lang}} <input class="input-xlarge" type="text" id="langs" value="{{.Lang}}" name="lang">
<input class="input-xlarge" type="text" id="langs" value="{{.}}" name="lang">
{{end}}
<input class="input-xlarge" type="text" id="langs" placeholder="Add langs" name="lang">
</div> </div>
</div> </div>
</fieldset> </fieldset>

2
templates/new.html
View file

@ -39,7 +39,7 @@
{{if .Subject}}<strong>Tags:</strong> {{range .Subject}}<a href="/search/?q=subject:{{.}}">{{.}}</a>, {{end}}<br />{{end}} {{if .Subject}}<strong>Tags:</strong> {{range .Subject}}<a href="/search/?q=subject:{{.}}">{{.}}</a>, {{end}}<br />{{end}}
{{if .Isbn}}<strong>ISBN:</strong> {{.Isbn}}<br />{{end}} {{if .Isbn}}<strong>ISBN:</strong> {{.Isbn}}<br />{{end}}
{{if .Date}}<strong>Date:</strong> {{.Date}}<br />{{end}} {{if .Date}}<strong>Date:</strong> {{.Date}}<br />{{end}}
{{if .Lang}}<strong>Lang:</strong> {{range .Lang}}<a href="/search/?q=lang:{{.}}">{{.}}</a> {{end}}<br />{{end}} {{if .Lang}}<strong>Lang:</strong> <a href="/search/?q=lang:{{.Lang}}">{{.Lang}}</a> <br />{{end}}
{{.Description}} {{.Description}}
</p> </p>
</div> </div>

2
templates/search.html
View file

@ -27,7 +27,7 @@
<div class="row"> <div class="row">
<div class="span7"> <div class="span7">
<p> <p>
<span class="muted">{{if .Lang}}{{.Lang}}{{end}}</span> <span class="muted">[{{if .Lang}}{{.Lang}}{{end}}]</span>
<a href="/book/{{.Id}}"><strong>{{.Title}}</strong></a> <a href="/book/{{.Id}}"><strong>{{.Title}}</strong></a>
<span class="muted">{{if .Publisher}}{{.Publisher}}{{end}}</span><br /> <span class="muted">{{if .Publisher}}{{.Publisher}}{{end}}</span><br />
{{range .Author}}{{.}}, {{end}} {{range .Author}}{{.}}, {{end}}

4
templates/search.opds
View file

@ -80,8 +80,8 @@
<dcterms:issued>{{.Date}}</dcterms:issued> <dcterms:issued>{{.Date}}</dcterms:issued>
{{end}} {{end}}
{{range .Lang}} {{if .Lang}}
<dcterms:language>{{.}}</dcterms:language> <dcterms:language>{{.Lang}}</dcterms:language>
{{end}} {{end}}
{{range .Subject}} {{range .Subject}}
<category term="{{html .}}" <category term="{{html .}}"