archives/trantor
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Move files to a local folder

Browse source
This commit is contained in:
Las Zenow 2014-08-21 19:24:23 -05:00
parent 97ed9ec073
commit f4690b2bba
18 changed files with 556 additions and 288 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1,4 +1,5 @@
trantor.git
store/
tools/adduser/adduser
tools/update/update
tools/togridfs/togridfs

14
admin.go
View file

@ -25,7 +25,7 @@ func deleteHandler(h handler) {
h.sess.Notify("Book not found!", "The book with id '"+id+"' is not there", "error")
continue
}
DeleteBook(book, h.db)
h.store.Delete(id)
h.db.DeleteBook(id)
if !book.Active {
@ -101,7 +101,7 @@ func saveHandler(h handler) {
h.sess.Notify("Book Modified!", "", "success")
h.sess.Save(h.w, h.r)
if h.db.BookActive(id) {
if h.db.IsBookActive(id) {
http.Redirect(h.w, h.r, "/book/"+id, http.StatusFound)
} else {
http.Redirect(h.w, h.r, "/new/", http.StatusFound)
@ -174,17 +174,25 @@ func storeHandler(h handler) {
var titles []string
ids := strings.Split(mux.Vars(h.r)["ids"], "/")
for _, id := range ids {
if id == "" {
continue
}
book, err := h.db.GetBookId(id)
if err != nil {
h.sess.Notify("Book not found!", "The book with id '"+id+"' is not there", "error")
continue
}
if err != nil {
h.sess.Notify("An error ocurred!", err.Error(), "error")
log.Error("Error getting book for storing '", book.Title, "': ", err.Error())
continue
}
err = h.db.ActiveBook(id)
if err != nil {
h.sess.Notify("An error ocurred!", err.Error(), "error")
log.Error("Error storing book '", book.Title, "': ", err.Error())
continue
}
h.db.UpdateBook(id, map[string]interface{}{"active": true})
titles = append(titles, book.Title)
}
if titles != nil {

7
config.go
View file

@ -6,8 +6,10 @@ const (
DB_IP = "127.0.0.1"
DB_NAME = "trantor"
META_COLL = "meta"
FS_BOOKS = "fs_books"
FS_IMGS = "fs_imgs"
EPUB_FILE = "book.epub"
COVER_FILE = "cover.jpg"
COVER_SMALL_FILE = "coverSmall.jpg"
MINUTES_UPDATE_TAGS = 11
MINUTES_UPDATE_VISITED = 41
@ -25,6 +27,7 @@ const (
NUM_NEWS = 10
DAYS_NEWS_INDEXPAGE = 15
STORE_PATH = "store/"
TEMPLATE_PATH = "templates/"
CSS_PATH = "css/"
JS_PATH = "js/"

79
cover.go
View file

@ -8,15 +8,13 @@ import _ "image/gif"
import (
"bytes"
"git.gitorious.org/go-pkg/epubgo.git"
"git.gitorious.org/trantor/trantor.git/database"
"git.gitorious.org/trantor/trantor.git/storage"
"github.com/gorilla/mux"
"github.com/nfnt/resize"
"image"
"image/jpeg"
"io"
"io/ioutil"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"regexp"
"strings"
)
@ -36,13 +34,11 @@ func coverHandler(h handler) {
}
}
fs := h.db.GetFS(FS_IMGS)
var f *mgo.GridFile
file := COVER_FILE
if vars["size"] == "small" {
f, err = fs.OpenId(book.CoverSmall)
} else {
f, err = fs.OpenId(book.Cover)
file = COVER_SMALL_FILE
}
f, err := h.store.Get(book.Id, file)
if err != nil {
log.Error("Error while opening image: ", err)
notFound(h)
@ -53,18 +49,21 @@ func coverHandler(h handler) {
headers := h.w.Header()
headers["Content-Type"] = []string{"image/jpeg"}
io.Copy(h.w, f)
_, err = io.Copy(h.w, f)
if err != nil {
log.Error("Error while copying image: ", err)
notFound(h)
return
}
}
func GetCover(e *epubgo.Epub, title string, db *database.DB) (bson.ObjectId, bson.ObjectId) {
imgId, smallId := coverFromMetadata(e, title, db)
if imgId != "" {
return imgId, smallId
func GetCover(e *epubgo.Epub, id string, store *storage.Store) bool {
if coverFromMetadata(e, id, store) {
return true
}
imgId, smallId = searchCommonCoverNames(e, title, db)
if imgId != "" {
return imgId, smallId
if searchCommonCoverNames(e, id, store) {
return true
}
/* search for img on the text */
@ -103,52 +102,52 @@ func GetCover(e *epubgo.Epub, title string, db *database.DB) (bson.ObjectId, bso
img, err := e.OpenFile(url)
if err == nil {
defer img.Close()
return storeImg(img, title, db)
return storeImg(img, id, store)
}
}
errNext = it.Next()
}
return "", ""
return false
}
func coverFromMetadata(e *epubgo.Epub, title string, db *database.DB) (bson.ObjectId, bson.ObjectId) {
func coverFromMetadata(e *epubgo.Epub, id string, store *storage.Store) bool {
metaList, _ := e.MetadataAttr("meta")
for _, meta := range metaList {
if meta["name"] == "cover" {
img, err := e.OpenFileId(meta["content"])
if err == nil {
defer img.Close()
return storeImg(img, title, db)
return storeImg(img, id, store)
}
}
}
return "", ""
return false
}
func searchCommonCoverNames(e *epubgo.Epub, title string, db *database.DB) (bson.ObjectId, bson.ObjectId) {
func searchCommonCoverNames(e *epubgo.Epub, id string, store *storage.Store) bool {
for _, p := range []string{"cover.jpg", "Images/cover.jpg", "images/cover.jpg", "cover.jpeg", "cover1.jpg", "cover1.jpeg"} {
img, err := e.OpenFile(p)
if err == nil {
defer img.Close()
return storeImg(img, title, db)
return storeImg(img, id, store)
}
}
return "", ""
return false
}
func storeImg(img io.Reader, title string, db *database.DB) (bson.ObjectId, bson.ObjectId) {
func storeImg(img io.Reader, id string, store *storage.Store) bool {
/* open the files */
fBig, err := createCoverFile(title, db)
fBig, err := store.Create(id, COVER_FILE)
if err != nil {
log.Error("Error creating ", title, ": ", err.Error())
return "", ""
log.Error("Error creating cover ", id, ": ", err.Error())
return false
}
defer fBig.Close()
fSmall, err := createCoverFile(title+"_small", db)
fSmall, err := store.Create(id, COVER_SMALL_FILE)
if err != nil {
log.Error("Error creating ", title+"_small", ": ", err.Error())
return "", ""
log.Error("Error creating small cover ", id, ": ", err.Error())
return false
}
defer fSmall.Close()
@ -159,32 +158,24 @@ func storeImg(img io.Reader, title string, db *database.DB) (bson.ObjectId, bson
imgResized, err := resizeImg(img1, IMG_WIDTH_BIG)
if err != nil {
log.Error("Error resizing big image: ", err.Error())
return "", ""
return false
}
err = jpeg.Encode(fBig, imgResized, &jpgOptions)
if err != nil {
log.Error("Error encoding big image: ", err.Error())
return "", ""
return false
}
imgSmallResized, err := resizeImg(&img2, IMG_WIDTH_SMALL)
if err != nil {
log.Error("Error resizing small image: ", err.Error())
return "", ""
return false
}
err = jpeg.Encode(fSmall, imgSmallResized, &jpgOptions)
if err != nil {
log.Error("Error encoding small image: ", err.Error())
return "", ""
return false
}
idBig, _ := fBig.Id().(bson.ObjectId)
idSmall, _ := fSmall.Id().(bson.ObjectId)
return idBig, idSmall
}
func createCoverFile(title string, db *database.DB) (*mgo.GridFile, error) {
fs := db.GetFS(FS_IMGS)
return fs.Create(title + ".jpg")
return true
}
func resizeImg(imgReader io.Reader, width uint) (image.Image, error) {

69
database/books.go
View file

@ -1,7 +1,6 @@
package database
import (
"errors"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
"gopkgs.com/unidecode.v1"
@ -14,7 +13,7 @@ const (
)
type Book struct {
Id string `bson:"_id"`
Id string
Title string
Author []string
Contributor string
@ -31,10 +30,8 @@ type Book struct {
Coverage string
Rights string
Meta string
File bson.ObjectId
FileSize int
Cover bson.ObjectId
CoverSmall bson.ObjectId
Cover bool
Active bool
Keywords []string
}
@ -66,35 +63,41 @@ func _getBooks(coll *mgo.Collection, query bson.M, length int, start int) (books
}
err = q.All(&books)
for i, b := range books {
books[i].Id = bson.ObjectId(b.Id).Hex()
}
return
}
func getBookId(coll *mgo.Collection, id string) (Book, error) {
var book Book
if !bson.IsObjectIdHex(id) {
return book, errors.New("Not valid book id")
}
err := coll.FindId(bson.ObjectIdHex(id)).One(&book)
book.Id = bson.ObjectId(book.Id).Hex()
err := coll.Find(bson.M{"id": id}).One(&book)
return book, err
}
func deleteBook(coll *mgo.Collection, id string) error {
return coll.RemoveId(bson.ObjectIdHex(id))
return coll.Remove(bson.M{"id": id})
}
func updateBook(coll *mgo.Collection, id string, data map[string]interface{}) error {
data["keywords"] = keywords(data)
return coll.UpdateId(bson.ObjectIdHex(id), bson.M{"$set": data})
var book map[string]interface{}
err := coll.Find(bson.M{"id": id}).One(&book)
if err != nil {
return err
}
for k, v := range data {
book[k] = v
}
data["keywords"] = keywords(book)
return coll.Update(bson.M{"id": id}, bson.M{"$set": data})
}
func bookActive(coll *mgo.Collection, id string) bool {
func activeBook(coll *mgo.Collection, id string) error {
data := map[string]interface{}{"active": true}
return coll.Update(bson.M{"id": id}, bson.M{"$set": data})
}
func isBookActive(coll *mgo.Collection, id string) bool {
var book Book
err := coll.FindId(bson.ObjectIdHex(id)).One(&book)
err := coll.Find(bson.M{"id": id}).One(&book)
if err != nil {
return false
}
@ -123,15 +126,29 @@ func buildQuery(q string) bson.M {
func keywords(b map[string]interface{}) (k []string) {
title, _ := b["title"].(string)
k = tokens(title)
author, _ := b["author"].([]string)
for _, a := range author {
k = append(k, tokens(a)...)
}
k = append(k, listKeywords(b["author"])...)
publisher, _ := b["publisher"].(string)
k = append(k, tokens(publisher)...)
subject, _ := b["subject"].([]string)
for _, s := range subject {
k = append(k, tokens(s)...)
k = append(k, listKeywords(b["subject"])...)
return
}
func listKeywords(v interface{}) (k []string) {
list, ok := v.([]string)
if !ok {
list, _ := v.([]interface{})
for _, e := range list {
str := e.(string)
k = append(k, tokens(str)...)
}
return
}
for _, e := range list {
k = append(k, tokens(e)...)
}
return
}

55
database/books_test.go
View file

@ -5,6 +5,7 @@ import "testing"
var book = map[string]interface{}{
"title": "some title",
"author": []string{"Alice", "Bob"},
"id": "r_m-IOzzIbA6QK5w",
}
func TestAddBook(t *testing.T) {
@ -15,7 +16,7 @@ func TestAddBook(t *testing.T) {
books, num, err := db.GetNewBooks(1, 0)
if err != nil {
t.Fatalf("db.GetBooks() return an error: ", err)
t.Fatal("db.GetBooks() return an error: ", err)
}
if num < 1 {
t.Fatalf("db.GetBooks() didn't find any result.")
@ -24,13 +25,61 @@ func TestAddBook(t *testing.T) {
t.Fatalf("db.GetBooks() didn't return any result.")
}
if books[0].Title != book["title"] {
t.Errorf("Book title don't match : '", books[0].Title, "' <=> '", book["title"], "'")
t.Error("Book title don't match : '", books[0].Title, "' <=> '", book["title"], "'")
}
}
func TestActiveBook(t *testing.T) {
db := Init(test_host, test_coll)
defer db.del()
tAddBook(t, db)
books, _, _ := db.GetNewBooks(1, 0)
id := books[0].Id
err := db.ActiveBook(id)
if err != nil {
t.Fatal("db.ActiveBook(", id, ") return an error: ", err)
}
b, err := db.GetBookId(id)
if err != nil {
t.Fatal("db.GetBookId(", id, ") return an error: ", err)
}
if b.Author[0] != books[0].Author[0] {
t.Error("Book author don't match : '", b.Author, "' <=> '", book["author"], "'")
}
}
func TestUpdateBookKeywords(t *testing.T) {
db := Init(test_host, test_coll)
defer db.del()
tAddBook(t, db)
books, _, _ := db.GetNewBooks(1, 0)
db.UpdateBook(books[0].Id, map[string]interface{}{"title": "Some other title"})
books, _, _ = db.GetNewBooks(1, 0)
keywords := books[0].Keywords
alice := false
bob := false
for _, e := range keywords {
if e == "alice" {
alice = true
}
if e == "bob" {
bob = true
}
}
if !alice || !bob {
t.Error("Alce or Bob are not in the keywords:", keywords)
}
}
func tAddBook(t *testing.T, db *DB) {
err := db.AddBook(book)
if err != nil {
t.Errorf("db.AddBook(", book, ") return an error: ", err)
t.Error("db.AddBook(", book, ") return an error: ", err)
}
}

13
database/database.go
View file

@ -73,9 +73,14 @@ func (db *DB) UpdateBook(id string, data map[string]interface{}) error {
return updateBook(booksColl, id, data)
}
func (db *DB) BookActive(id string) bool {
func (db *DB) ActiveBook(id string) error {
booksColl := db.session.DB(db.name).C(books_coll)
return bookActive(booksColl, id)
return activeBook(booksColl, id)
}
func (db *DB) IsBookActive(id string) bool {
booksColl := db.session.DB(db.name).C(books_coll)
return isBookActive(booksColl, id)
}
func (db *DB) User(name string) *User {
@ -154,10 +159,6 @@ func (db *DB) UpdateDownloadedBooks() error {
return u.UpdateMostBooks("download")
}
func (db *DB) GetFS(prefix string) *mgo.GridFS {
return db.session.DB(db.name).GridFS(prefix)
}
func (db *DB) GetTags() ([]string, error) {
tagsColl := db.session.DB(db.name).C(tags_coll)
return GetTags(tagsColl)

29
reader.go
View file

@ -1,11 +1,15 @@
package main
import log "github.com/cihub/seelog"
import (
"bytes"
"git.gitorious.org/go-pkg/epubgo.git"
"git.gitorious.org/trantor/trantor.git/database"
"git.gitorious.org/trantor/trantor.git/storage"
"github.com/gorilla/mux"
"io"
"labix.org/v2/mgo/bson"
"io/ioutil"
"net/http"
"strconv"
"strings"
@ -134,6 +138,7 @@ func readStartHandler(h handler) {
id := mux.Vars(h.r)["id"]
e, _ := openReadEpub(h)
if e == nil {
log.Warn("Open epub returns an empty file")
notFound(h)
return
}
@ -141,6 +146,7 @@ func readStartHandler(h handler) {
it, err := e.Spine()
if err != nil {
log.Warn("No spine in the epub")
notFound(h)
return
}
@ -175,7 +181,7 @@ func readHandler(h handler) {
func openReadEpub(h handler) (*epubgo.Epub, database.Book) {
var book database.Book
id := mux.Vars(h.r)["id"]
if !bson.IsObjectIdHex(id) {
if id == "" {
return nil, book
}
book, err := h.db.GetBookId(id)
@ -188,7 +194,7 @@ func openReadEpub(h handler) (*epubgo.Epub, database.Book) {
return nil, book
}
}
e, err := OpenBook(book.File, h.db)
e, err := openBook(book.Id, h.store)
if err != nil {
return nil, book
}
@ -199,7 +205,7 @@ func contentHandler(h handler) {
vars := mux.Vars(h.r)
id := vars["id"]
file := vars["file"]
if file == "" || !bson.IsObjectIdHex(id) {
if file == "" || id == "" {
notFound(h)
return
}
@ -215,7 +221,7 @@ func contentHandler(h handler) {
return
}
}
e, err := OpenBook(book.File, h.db)
e, err := openBook(book.Id, h.store)
if err != nil {
notFound(h)
return
@ -230,3 +236,16 @@ func contentHandler(h handler) {
defer html.Close()
io.Copy(h.w, html)
}
func openBook(id string, store *storage.Store) (*epubgo.Epub, error) {
f, err := store.Get(id, EPUB_FILE)
if err != nil {
return nil, err
}
defer f.Close()
buff, err := ioutil.ReadAll(f)
reader := bytes.NewReader(buff)
return epubgo.Load(reader, int64(len(buff)))
}

58
stats.go
View file

@ -4,32 +4,49 @@ import log "github.com/cihub/seelog"
import (
"git.gitorious.org/trantor/trantor.git/database"
"git.gitorious.org/trantor/trantor.git/storage"
"github.com/gorilla/mux"
"labix.org/v2/mgo/bson"
"net/http"
"strconv"
"strings"
"time"
)
const (
stats_version = 2
)
type handler struct {
w http.ResponseWriter
r *http.Request
sess *Session
db *database.DB
w http.ResponseWriter
r *http.Request
sess *Session
db *database.DB
store *storage.Store
}
func InitStats(database *database.DB) {
statsChannel = make(chan statsRequest, CHAN_SIZE)
go statsWorker(database)
type StatsGatherer struct {
db *database.DB
store *storage.Store
channel chan statsRequest
}
func GatherStats(function func(handler), database *database.DB) func(http.ResponseWriter, *http.Request) {
func InitStats(database *database.DB, store *storage.Store) *StatsGatherer {
sg := new(StatsGatherer)
sg.channel = make(chan statsRequest, CHAN_SIZE)
sg.db = database
sg.store = store
go sg.worker()
return sg
}
func (sg StatsGatherer) Gather(function func(handler)) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
log.Info("Query ", r.Method, " ", r.RequestURI)
var h handler
h.db = database.Copy()
h.store = sg.store
h.db = sg.db.Copy()
defer h.db.Close()
h.w = w
@ -37,12 +54,10 @@ func GatherStats(function func(handler), database *database.DB) func(http.Respon
h.sess = GetSession(r, h.db)
function(h)
statsChannel <- statsRequest{bson.Now(), mux.Vars(r), h.sess, r}
sg.channel <- statsRequest{time.Now(), mux.Vars(r), h.sess, r}
}
}
var statsChannel chan statsRequest
type statsRequest struct {
date time.Time
vars map[string]string
@ -50,16 +65,17 @@ type statsRequest struct {
r *http.Request
}
func statsWorker(database *database.DB) {
db := database.Copy()
func (sg StatsGatherer) worker() {
db := sg.db.Copy()
defer db.Close()
for req := range statsChannel {
for req := range sg.channel {
stats := make(map[string]interface{})
appendFiles(req.r, stats)
appendMuxVars(req.vars, stats)
appendUrl(req.r, stats)
appendSession(req.sess, stats)
stats["version"] = stats_version
stats["method"] = req.r.Method
stats["date"] = req.date
db.AddStats(stats)
@ -141,16 +157,12 @@ func appendMuxVars(vars map[string]string, stats map[string]interface{}) {
for key, value := range vars {
switch {
case key == "id":
if bson.IsObjectIdHex(value) {
stats["id"] = bson.ObjectIdHex(value)
}
stats["id"] = value
case key == "ids":
var objectIds []bson.ObjectId
var objectIds []string
ids := strings.Split(value, "/")
for _, id := range ids {
if bson.IsObjectIdHex(value) {
objectIds = append(objectIds, bson.ObjectIdHex(id))
}
objectIds = append(objectIds, id)
}
if len(objectIds) > 0 {
stats["ids"] = objectIds

42
storage/dir.go Normal file
View file

@ -0,0 +1,42 @@
package storage
import p "path"
import (
"os"
)
const (
dir_depth = 2
encodeURL = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
)
func mkstore(path string) error {
return _mkstore(path, dir_depth)
}
func _mkstore(path string, depth int) error {
err := os.MkdirAll(path, os.ModePerm)
if err != nil || depth == 0 {
return err
}
for _, l := range encodeURL {
next_path := p.Join(path, string(l))
err = _mkstore(next_path, depth-1)
if err != nil {
return err
}
}
return nil
}
func idPath(storePath string, id string) string {
path := storePath
for i := 0; i < dir_depth; i++ {
dir := string(id[i])
path = p.Join(path, dir)
}
path = p.Join(path, id)
return path
}

57
storage/storage.go Normal file
View file

@ -0,0 +1,57 @@
package storage
import p "path"
import (
"io"
"os"
)
type Store struct {
path string
}
func Init(path string) (*Store, error) {
st := new(Store)
st.path = path
_, err := os.Stat(path)
if err != nil {
err = mkstore(st.path)
}
return st, err
}
func (st *Store) Create(id string, name string) (io.WriteCloser, error) {
path := idPath(st.path, id)
err := os.MkdirAll(path, os.ModePerm)
if err != nil {
return nil, err
}
return os.Create(p.Join(path, name))
}
func (st *Store) Store(id string, file io.Reader, name string) (size int64, err error) {
dest, err := st.Create(id, name)
if err != nil {
return 0, err
}
defer dest.Close()
return io.Copy(dest, file)
}
func (st *Store) Get(id string, name string) (io.ReadCloser, error) {
path := idPath(st.path, id)
return os.Open(p.Join(path, name))
}
func (st *Store) Delete(id string) error {
path := idPath(st.path, id)
return os.RemoveAll(path)
}
func (st *Store) del() {
os.RemoveAll(st.path)
}

113
storage/storage_test.go Normal file
View file

@ -0,0 +1,113 @@
package storage
import "testing"
import (
"bytes"
"io"
"io/ioutil"
"os"
"strings"
)
const (
test_path = "/tmp/store"
test_book = `
HARI SELDON-… born in the 11,988th year of the Galactic Era; died
12,069. The dates are more commonly given in terms of the current
Foundational Era as - 79 to the year 1 F.E. Born to middle-class
parents on Helicon, Arcturus sector (where his father, in a legend of
doubtful authenticity, was a tobacco grower in the hydroponic plants
of the planet), he early showed amazing ability in mathematics.
Anecdotes concerning his ability are innumerable, and some are
contradictory. At the age of two, he is said to have …`
test_id = "1234567890abcdef"
)
func TestInit(t *testing.T) {
st, err := Init(test_path)
if err != nil {
t.Fatal("An error ocurred initializing the store =>", err)
}
defer st.del()
info, err := os.Stat(test_path)
if err != nil {
t.Fatal("An error ocurred =>", err)
}
if !info.Mode().IsDir() {
t.Errorf(test_path, " is not dir.")
}
info, err = os.Stat(test_path + "/a/M")
if err != nil {
t.Fatal("An error ocurred =>", err)
}
if !info.Mode().IsDir() {
t.Errorf(test_path, " is not dir.")
}
}
func TestStore(t *testing.T) {
st, err := Init(test_path)
defer st.del()
_, err = st.Store(test_id, strings.NewReader(test_book), "epub")
if err != nil {
t.Fatal("An error ocurred storing the book =>", err)
}
book, err := st.Get(test_id, "epub")
if err != nil {
t.Fatal("An error ocurred getting the book =>", err)
}
content, err := ioutil.ReadAll(book)
if err != nil {
t.Fatal("An error ocurred reading the book =>", err)
}
if !bytes.Equal(content, []byte(test_book)) {
t.Error("Not the same content")
}
}
func TestCreate(t *testing.T) {
st, err := Init(test_path)
defer st.del()
f, err := st.Create(test_id, "img")
if err != nil {
t.Fatal("An error ocurred storing the book =>", err)
}
io.Copy(f, strings.NewReader(test_book))
img, err := st.Get(test_id, "img")
if err != nil {
t.Fatal("An error ocurred getting the book =>", err)
}
content, err := ioutil.ReadAll(img)
if err != nil {
t.Fatal("An error ocurred reading the book =>", err)
}
if !bytes.Equal(content, []byte(test_book)) {
t.Error("Not the same content")
}
}
func TestDelete(t *testing.T) {
st, err := Init(test_path)
defer st.del()
_, err = st.Store(test_id, strings.NewReader(test_book), "epub")
if err != nil {
t.Fatal("An error ocurred storing the book =>", err)
}
err = st.Delete(test_id)
if err != nil {
t.Fatal("An error ocurred deleteing id =>", err)
}
_, err = st.Get(test_id, "epub")
if err == nil {
t.Fatal("Retrieve book without error.")
}
}

115
store.go
View file

@ -1,115 +0,0 @@
package main
import (
"bytes"
"git.gitorious.org/go-pkg/epubgo.git"
"git.gitorious.org/trantor/trantor.git/database"
"io"
"io/ioutil"
"labix.org/v2/mgo/bson"
"regexp"
"strings"
)
func OpenBook(id bson.ObjectId, db *database.DB) (*epubgo.Epub, error) {
fs := db.GetFS(FS_BOOKS)
f, err := fs.OpenId(id)
if err != nil {
return nil, err
}
defer f.Close()
buff, err := ioutil.ReadAll(f)
reader := bytes.NewReader(buff)
return epubgo.Load(reader, int64(len(buff)))
}
func StoreNewFile(name string, file io.Reader, db *database.DB) (bson.ObjectId, int64, error) {
fs := db.GetFS(FS_BOOKS)
fw, err := fs.Create(name)
if err != nil {
return "", 0, err
}
defer fw.Close()
size, err := io.Copy(fw, file)
id, _ := fw.Id().(bson.ObjectId)
return id, size, err
}
func DeleteFile(id bson.ObjectId, db *database.DB) error {
fs := db.GetFS(FS_BOOKS)
return fs.RemoveId(id)
}
func DeleteCover(id bson.ObjectId, db *database.DB) error {
fs := db.GetFS(FS_IMGS)
return fs.RemoveId(id)
}
func DeleteBook(book database.Book, db *database.DB) {
if book.Cover != "" {
DeleteCover(book.Cover, db)
}
if book.CoverSmall != "" {
DeleteCover(book.CoverSmall, db)
}
DeleteFile(book.File, db)
}
func cleanStr(str string) string {
str = strings.Replace(str, "&#39;", "'", -1)
exp, _ := regexp.Compile("&[^;]*;")
str = exp.ReplaceAllString(str, "")
exp, _ = regexp.Compile("[ ,]*$")
str = exp.ReplaceAllString(str, "")
return str
}
func parseAuthr(creator []string) []string {
exp1, _ := regexp.Compile("^(.*\\( *([^\\)]*) *\\))*$")
exp2, _ := regexp.Compile("^[^:]*: *(.*)$")
res := make([]string, len(creator))
for i, s := range creator {
auth := exp1.FindStringSubmatch(s)
if auth != nil {
res[i] = cleanStr(strings.Join(auth[2:], ", "))
} else {
auth := exp2.FindStringSubmatch(s)
if auth != nil {
res[i] = cleanStr(auth[1])
} else {
res[i] = cleanStr(s)
}
}
}
return res
}
func parseDescription(description []string) string {
str := cleanStr(strings.Join(description, "\n"))
str = strings.Replace(str, "</p>", "\n", -1)
exp, _ := regexp.Compile("<[^>]*>")
str = exp.ReplaceAllString(str, "")
str = strings.Replace(str, "&amp;", "&", -1)
str = strings.Replace(str, "&lt;", "<", -1)
str = strings.Replace(str, "&gt;", ">", -1)
str = strings.Replace(str, "\\n", "\n", -1)
return str
}
func parseSubject(subject []string) []string {
var res []string
for _, s := range subject {
res = append(res, strings.Split(s, " / ")...)
}
return res
}
func parseDate(date []string) string {
if len(date) == 0 {
return ""
}
return strings.Replace(date[0], "Unspecified: ", "", -1)
}

6
templates/index.html
View file

@ -26,7 +26,7 @@
<li class="span2">
<div class="thumbnail centered" style="border:none;">
<a href="/book/{{.Id}}">
{{if .CoverSmall}}<div class="down"><img class="img-rounded" src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" /></div>{{end}}
{{if .Cover}}<div class="down"><img class="img-rounded" src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" /></div>{{end}}
<p><strong>{{.Title}}</strong></p>
</a>
</div>
@ -44,7 +44,7 @@
<li class="span2">
<div class="thumbnail centered" style="border:none;">
<a href="/book/{{.Id}}">
{{if .CoverSmall}}<div class="down"><img class="img-rounded" src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" /></div>{{end}}
{{if .Cover}}<div class="down"><img class="img-rounded" src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" /></div>{{end}}
<p><strong>{{.Title}}</strong></p>
</a>
</div>
@ -62,7 +62,7 @@
<li class="span2">
<div class="thumbnail centered" style="border:none;">
<a href="/book/{{.Id}}">
{{if .CoverSmall}}<div class="down"><img class="img-rounded" src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" /></div>{{end}}
{{if .Cover}}<div class="down"><img class="img-rounded" src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" /></div>{{end}}
<p><strong>{{.Title}}</strong></p>
</a>
</div>

2
templates/new.html
View file

@ -26,7 +26,7 @@
{{with .B}}
<div class="row">
<div class="span1">
<p class="pull-right">{{if .CoverSmall}}<img src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" />{{end}}</p>
<p class="pull-right">{{if .Cover}}<img src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" />{{end}}</p>
</div>
<div class="span9">
<p><a href="/search/?q=title:{{.Title}}"><strong>{{.Title}}</strong></a> <small>({{$titleFound}})</small><br />

2
templates/search.html
View file

@ -21,7 +21,7 @@
{{range .}}
<div class="row">
<div class="span1">
<p class="pull-right"><a href="/book/{{.Id}}">{{if .CoverSmall}}<img class="img-rounded" src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" />{{end}}</a></p>
<p class="pull-right"><a href="/book/{{.Id}}">{{if .Cover}}<img class="img-rounded" src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" />{{end}}</a></p>
</div>
<div class="span10 well">
<div class="row">

81
trantor.go
View file

@ -4,9 +4,11 @@ import log "github.com/cihub/seelog"
import (
"git.gitorious.org/trantor/trantor.git/database"
"git.gitorious.org/trantor/trantor.git/storage"
"github.com/gorilla/mux"
"io"
"net/http"
"os"
"strings"
)
@ -71,8 +73,7 @@ func downloadHandler(h handler) {
}
}
fs := h.db.GetFS(FS_BOOKS)
f, err := fs.OpenId(book.File)
f, err := h.store.Get(book.Id, EPUB_FILE)
if err != nil {
notFound(h)
return
@ -81,7 +82,7 @@ func downloadHandler(h handler) {
headers := h.w.Header()
headers["Content-Type"] = []string{"application/epub+zip"}
headers["Content-Disposition"] = []string{"attachment; filename=\"" + f.Name() + "\""}
headers["Content-Disposition"] = []string{"attachment; filename=\"" + book.Title + ".epub\""}
io.Copy(h.w, f)
}
@ -137,47 +138,55 @@ func main() {
db := database.Init(DB_IP, DB_NAME)
defer db.Close()
InitTasks(db)
InitStats(db)
InitUpload(db)
store, err := storage.Init(STORE_PATH)
if err != nil {
log.Critical("Problem initializing store: ", err)
os.Exit(1)
}
initRouter(db)
InitTasks(db)
sg := InitStats(db, store)
InitUpload(db, store)
initRouter(db, sg)
log.Error(http.ListenAndServe(":"+PORT, nil))
}
func initRouter(db *database.DB) {
func initRouter(db *database.DB, sg *StatsGatherer) {
const id_pattern = "[0-9a-zA-Z\\-\\_]{16}"
r := mux.NewRouter()
var notFoundHandler http.HandlerFunc
notFoundHandler = GatherStats(notFound, db)
notFoundHandler = sg.Gather(notFound)
r.NotFoundHandler = notFoundHandler
r.HandleFunc("/", GatherStats(indexHandler, db))
r.HandleFunc("/book/{id:[0-9a-fA-F]+}", GatherStats(bookHandler, db))
r.HandleFunc("/search/", GatherStats(searchHandler, db))
r.HandleFunc("/upload/", GatherStats(uploadHandler, db)).Methods("GET")
r.HandleFunc("/upload/", GatherStats(uploadPostHandler, db)).Methods("POST")
r.HandleFunc("/login/", GatherStats(loginHandler, db)).Methods("GET")
r.HandleFunc("/login/", GatherStats(loginPostHandler, db)).Methods("POST")
r.HandleFunc("/create_user/", GatherStats(createUserHandler, db)).Methods("POST")
r.HandleFunc("/logout/", GatherStats(logoutHandler, db))
r.HandleFunc("/new/", GatherStats(newHandler, db))
r.HandleFunc("/store/{ids:([0-9a-fA-F]+/)+}", GatherStats(storeHandler, db))
r.HandleFunc("/delete/{ids:([0-9a-fA-F]+/)+}", GatherStats(deleteHandler, db))
r.HandleFunc("/read/{id:[0-9a-fA-F]+}", GatherStats(readStartHandler, db))
r.HandleFunc("/read/{id:[0-9a-fA-F]+}/{file:.*}", GatherStats(readHandler, db))
r.HandleFunc("/content/{id:[0-9a-fA-F]+}/{file:.*}", GatherStats(contentHandler, db))
r.HandleFunc("/edit/{id:[0-9a-fA-F]+}", GatherStats(editHandler, db))
r.HandleFunc("/save/{id:[0-9a-fA-F]+}", GatherStats(saveHandler, db)).Methods("POST")
r.HandleFunc("/about/", GatherStats(aboutHandler, db))
r.HandleFunc("/help/", GatherStats(helpHandler, db))
r.HandleFunc("/download/{id:[0-9a-fA-F]+}/{epub:.*}", GatherStats(downloadHandler, db))
r.HandleFunc("/cover/{id:[0-9a-fA-F]+}/{size}/{img:.*}", GatherStats(coverHandler, db))
r.HandleFunc("/dashboard/", GatherStats(dashboardHandler, db))
r.HandleFunc("/settings/", GatherStats(settingsHandler, db))
r.HandleFunc("/stats/", GatherStats(statsHandler, db))
r.HandleFunc("/news/", GatherStats(newsHandler, db))
r.HandleFunc("/news/edit", GatherStats(editNewsHandler, db)).Methods("GET")
r.HandleFunc("/news/edit", GatherStats(postNewsHandler, db)).Methods("POST")
r.HandleFunc("/", sg.Gather(indexHandler))
r.HandleFunc("/book/{id:"+id_pattern+"}", sg.Gather(bookHandler))
r.HandleFunc("/search/", sg.Gather(searchHandler))
r.HandleFunc("/upload/", sg.Gather(uploadHandler)).Methods("GET")
r.HandleFunc("/upload/", sg.Gather(uploadPostHandler)).Methods("POST")
r.HandleFunc("/login/", sg.Gather(loginHandler)).Methods("GET")
r.HandleFunc("/login/", sg.Gather(loginPostHandler)).Methods("POST")
r.HandleFunc("/create_user/", sg.Gather(createUserHandler)).Methods("POST")
r.HandleFunc("/logout/", sg.Gather(logoutHandler))
r.HandleFunc("/new/", sg.Gather(newHandler))
r.HandleFunc("/store/{ids:("+id_pattern+"/)+}", sg.Gather(storeHandler))
r.HandleFunc("/delete/{ids:("+id_pattern+"/)+}", sg.Gather(deleteHandler))
r.HandleFunc("/read/{id:"+id_pattern+"}", sg.Gather(readStartHandler))
r.HandleFunc("/read/{id:"+id_pattern+"}/{file:.*}", sg.Gather(readHandler))
r.HandleFunc("/content/{id:"+id_pattern+"}/{file:.*}", sg.Gather(contentHandler))
r.HandleFunc("/edit/{id:"+id_pattern+"}", sg.Gather(editHandler))
r.HandleFunc("/save/{id:"+id_pattern+"}", sg.Gather(saveHandler)).Methods("POST")
r.HandleFunc("/about/", sg.Gather(aboutHandler))
r.HandleFunc("/help/", sg.Gather(helpHandler))
r.HandleFunc("/download/{id:"+id_pattern+"}/{epub:.*}", sg.Gather(downloadHandler))
r.HandleFunc("/cover/{id:"+id_pattern+"}/{size}/{img:.*}", sg.Gather(coverHandler))
r.HandleFunc("/dashboard/", sg.Gather(dashboardHandler))
r.HandleFunc("/settings/", sg.Gather(settingsHandler))
r.HandleFunc("/stats/", sg.Gather(statsHandler))
r.HandleFunc("/news/", sg.Gather(newsHandler))
r.HandleFunc("/news/edit", sg.Gather(editNewsHandler)).Methods("GET")
r.HandleFunc("/news/edit", sg.Gather(postNewsHandler)).Methods("POST")
h := http.FileServer(http.Dir(IMG_PATH))
r.Handle("/img/{img}", http.StripPrefix("/img/", h))
h = http.FileServer(http.Dir(CSS_PATH))

101
upload.go
View file

@ -4,16 +4,20 @@ import log "github.com/cihub/seelog"
import (
"bytes"
"crypto/rand"
"encoding/base64"
"git.gitorious.org/go-pkg/epubgo.git"
"git.gitorious.org/trantor/trantor.git/database"
"git.gitorious.org/trantor/trantor.git/storage"
"io/ioutil"
"mime/multipart"
"regexp"
"strings"
)
func InitUpload(database *database.DB) {
func InitUpload(database *database.DB, store *storage.Store) {
uploadChannel = make(chan uploadRequest, CHAN_SIZE)
go uploadWorker(database)
go uploadWorker(database, store)
}
var uploadChannel chan uploadRequest
@ -23,16 +27,16 @@ type uploadRequest struct {
filename string
}
func uploadWorker(database *database.DB) {
func uploadWorker(database *database.DB, store *storage.Store) {
db := database.Copy()
defer db.Close()
for req := range uploadChannel {
processFile(req, db)
processFile(req, db, store)
}
}
func processFile(req uploadRequest, db *database.DB) {
func processFile(req uploadRequest, db *database.DB, store *storage.Store) {
defer req.file.Close()
epub, err := openMultipartEpub(req.file)
@ -42,20 +46,18 @@ func processFile(req uploadRequest, db *database.DB) {
}
defer epub.Close()
book := parseFile(epub, db)
title, _ := book["title"].(string)
book, id := parseFile(epub, store)
req.file.Seek(0, 0)
id, size, err := StoreNewFile(title+".epub", req.file, db)
size, err := store.Store(id, req.file, EPUB_FILE)
if err != nil {
log.Error("Error storing book (", title, "): ", err)
log.Error("Error storing book (", id, "): ", err)
return
}
book["file"] = id
book["filesize"] = size
err = db.AddBook(book)
if err != nil {
log.Error("Error storing metadata (", title, "): ", err)
log.Error("Error storing metadata (", id, "): ", err)
return
}
log.Info("File uploaded: ", req.filename)
@ -104,7 +106,7 @@ func openMultipartEpub(file multipart.File) (*epubgo.Epub, error) {
return epubgo.Load(reader, int64(len(buff)))
}
func parseFile(epub *epubgo.Epub, db *database.DB) map[string]interface{} {
func parseFile(epub *epubgo.Epub, store *storage.Store) (metadata map[string]interface{}, id string) {
book := map[string]interface{}{}
for _, m := range epub.MetadataFields() {
data, err := epub.Metadata(m)
@ -135,12 +137,71 @@ func parseFile(epub *epubgo.Epub, db *database.DB) map[string]interface{} {
book[m] = strings.Join(data, ", ")
}
}
title, _ := book["title"].(string)
book["file"] = nil
cover, coverSmall := GetCover(epub, title, db)
if cover != "" {
book["cover"] = cover
book["coversmall"] = coverSmall
}
return book
id = genId()
book["id"] = id //TODO
book["cover"] = GetCover(epub, id, store)
return book, id
}
func genId() string {
b := make([]byte, 12)
rand.Read(b)
return base64.URLEncoding.EncodeToString(b)
}
func cleanStr(str string) string {
str = strings.Replace(str, "&#39;", "'", -1)
exp, _ := regexp.Compile("&[^;]*;")
str = exp.ReplaceAllString(str, "")
exp, _ = regexp.Compile("[ ,]*$")
str = exp.ReplaceAllString(str, "")
return str
}
func parseAuthr(creator []string) []string {
exp1, _ := regexp.Compile("^(.*\\( *([^\\)]*) *\\))*$")
exp2, _ := regexp.Compile("^[^:]*: *(.*)$")
res := make([]string, len(creator))
for i, s := range creator {
auth := exp1.FindStringSubmatch(s)
if auth != nil {
res[i] = cleanStr(strings.Join(auth[2:], ", "))
} else {
auth := exp2.FindStringSubmatch(s)
if auth != nil {
res[i] = cleanStr(auth[1])
} else {
res[i] = cleanStr(s)
}
}
}
return res
}
func parseDescription(description []string) string {
str := cleanStr(strings.Join(description, "\n"))
str = strings.Replace(str, "</p>", "\n", -1)
exp, _ := regexp.Compile("<[^>]*>")
str = exp.ReplaceAllString(str, "")
str = strings.Replace(str, "&amp;", "&", -1)
str = strings.Replace(str, "&lt;", "<", -1)
str = strings.Replace(str, "&gt;", ">", -1)
str = strings.Replace(str, "\\n", "\n", -1)
return str
}
func parseSubject(subject []string) []string {
var res []string
for _, s := range subject {
res = append(res, strings.Split(s, " / ")...)
}
return res
}
func parseDate(date []string) string {
if len(date) == 0 {
return ""
}
return strings.Replace(date[0], "Unspecified: ", "", -1)
}