Move files to a local folder

.gitignore

@@ -1,4 +1,5 @@
 trantor.git
+store/
 tools/adduser/adduser
 tools/update/update
 tools/togridfs/togridfs

admin.go

@@ -25,7 +25,7 @@ func deleteHandler(h handler) {
 			h.sess.Notify("Book not found!", "The book with id '"+id+"' is not there", "error")
 			continue
 		}
-		DeleteBook(book, h.db)
+		h.store.Delete(id)
 		h.db.DeleteBook(id)
 
 		if !book.Active {
@@ -101,7 +101,7 @@ func saveHandler(h handler) {
 
 	h.sess.Notify("Book Modified!", "", "success")
 	h.sess.Save(h.w, h.r)
-	if h.db.BookActive(id) {
+	if h.db.IsBookActive(id) {
 		http.Redirect(h.w, h.r, "/book/"+id, http.StatusFound)
 	} else {
 		http.Redirect(h.w, h.r, "/new/", http.StatusFound)
@@ -174,17 +174,25 @@ func storeHandler(h handler) {
 	var titles []string
 	ids := strings.Split(mux.Vars(h.r)["ids"], "/")
 	for _, id := range ids {
+		if id == "" {
+			continue
+		}
 		book, err := h.db.GetBookId(id)
 		if err != nil {
 			h.sess.Notify("Book not found!", "The book with id '"+id+"' is not there", "error")
 			continue
 		}
+		if err != nil {
+			h.sess.Notify("An error ocurred!", err.Error(), "error")
+			log.Error("Error getting book for storing '", book.Title, "': ", err.Error())
+			continue
+		}
+		err = h.db.ActiveBook(id)
 		if err != nil {
 			h.sess.Notify("An error ocurred!", err.Error(), "error")
 			log.Error("Error storing book '", book.Title, "': ", err.Error())
 			continue
 		}
-		h.db.UpdateBook(id, map[string]interface{}{"active": true})
 		titles = append(titles, book.Title)
 	}
 	if titles != nil {

config.go

@@ -6,8 +6,10 @@ const (
 	DB_IP     = "127.0.0.1"
 	DB_NAME   = "trantor"
 	META_COLL = "meta"
-	FS_BOOKS  = "fs_books"
+
-	FS_IMGS   = "fs_imgs"
+	EPUB_FILE        = "book.epub"
+	COVER_FILE       = "cover.jpg"
+	COVER_SMALL_FILE = "coverSmall.jpg"
 
 	MINUTES_UPDATE_TAGS       = 11
 	MINUTES_UPDATE_VISITED    = 41
@@ -25,6 +27,7 @@ const (
 	NUM_NEWS                  = 10
 	DAYS_NEWS_INDEXPAGE       = 15
 
+	STORE_PATH    = "store/"
 	TEMPLATE_PATH = "templates/"
 	CSS_PATH      = "css/"
 	JS_PATH       = "js/"

cover.go

@@ -8,15 +8,13 @@ import _ "image/gif"
 import (
 	"bytes"
 	"git.gitorious.org/go-pkg/epubgo.git"
-	"git.gitorious.org/trantor/trantor.git/database"
+	"git.gitorious.org/trantor/trantor.git/storage"
 	"github.com/gorilla/mux"
 	"github.com/nfnt/resize"
 	"image"
 	"image/jpeg"
 	"io"
 	"io/ioutil"
-	"labix.org/v2/mgo"
-	"labix.org/v2/mgo/bson"
 	"regexp"
 	"strings"
 )
@@ -36,13 +34,11 @@ func coverHandler(h handler) {
 		}
 	}
 
-	fs := h.db.GetFS(FS_IMGS)
+	file := COVER_FILE
-	var f *mgo.GridFile
 	if vars["size"] == "small" {
-		f, err = fs.OpenId(book.CoverSmall)
+		file = COVER_SMALL_FILE
-	} else {
-		f, err = fs.OpenId(book.Cover)
 	}
+	f, err := h.store.Get(book.Id, file)
 	if err != nil {
 		log.Error("Error while opening image: ", err)
 		notFound(h)
@@ -53,18 +49,21 @@ func coverHandler(h handler) {
 	headers := h.w.Header()
 	headers["Content-Type"] = []string{"image/jpeg"}
 
-	io.Copy(h.w, f)
+	_, err = io.Copy(h.w, f)
+	if err != nil {
+		log.Error("Error while copying image: ", err)
+		notFound(h)
+		return
+	}
 }
 
-func GetCover(e *epubgo.Epub, title string, db *database.DB) (bson.ObjectId, bson.ObjectId) {
+func GetCover(e *epubgo.Epub, id string, store *storage.Store) bool {
-	imgId, smallId := coverFromMetadata(e, title, db)
+	if coverFromMetadata(e, id, store) {
-	if imgId != "" {
+		return true
-		return imgId, smallId
 	}
 
-	imgId, smallId = searchCommonCoverNames(e, title, db)
+	if searchCommonCoverNames(e, id, store) {
-	if imgId != "" {
+		return true
-		return imgId, smallId
 	}
 
 	/* search for img on the text */
@@ -103,52 +102,52 @@ func GetCover(e *epubgo.Epub, title string, db *database.DB) (bson.ObjectId, bso
 			img, err := e.OpenFile(url)
 			if err == nil {
 				defer img.Close()
-				return storeImg(img, title, db)
+				return storeImg(img, id, store)
 			}
 		}
 		errNext = it.Next()
 	}
-	return "", ""
+	return false
 }
 
-func coverFromMetadata(e *epubgo.Epub, title string, db *database.DB) (bson.ObjectId, bson.ObjectId) {
+func coverFromMetadata(e *epubgo.Epub, id string, store *storage.Store) bool {
 	metaList, _ := e.MetadataAttr("meta")
 	for _, meta := range metaList {
 		if meta["name"] == "cover" {
 			img, err := e.OpenFileId(meta["content"])
 			if err == nil {
 				defer img.Close()
-				return storeImg(img, title, db)
+				return storeImg(img, id, store)
 			}
 		}
 	}
-	return "", ""
+	return false
 }
 
-func searchCommonCoverNames(e *epubgo.Epub, title string, db *database.DB) (bson.ObjectId, bson.ObjectId) {
+func searchCommonCoverNames(e *epubgo.Epub, id string, store *storage.Store) bool {
 	for _, p := range []string{"cover.jpg", "Images/cover.jpg", "images/cover.jpg", "cover.jpeg", "cover1.jpg", "cover1.jpeg"} {
 		img, err := e.OpenFile(p)
 		if err == nil {
 			defer img.Close()
-			return storeImg(img, title, db)
+			return storeImg(img, id, store)
 		}
 	}
-	return "", ""
+	return false
 }
 
-func storeImg(img io.Reader, title string, db *database.DB) (bson.ObjectId, bson.ObjectId) {
+func storeImg(img io.Reader, id string, store *storage.Store) bool {
 	/* open the files */
-	fBig, err := createCoverFile(title, db)
+	fBig, err := store.Create(id, COVER_FILE)
 	if err != nil {
-		log.Error("Error creating ", title, ": ", err.Error())
+		log.Error("Error creating cover ", id, ": ", err.Error())
-		return "", ""
+		return false
 	}
 	defer fBig.Close()
 
-	fSmall, err := createCoverFile(title+"_small", db)
+	fSmall, err := store.Create(id, COVER_SMALL_FILE)
 	if err != nil {
-		log.Error("Error creating ", title+"_small", ": ", err.Error())
+		log.Error("Error creating small cover ", id, ": ", err.Error())
-		return "", ""
+		return false
 	}
 	defer fSmall.Close()
 
@@ -159,32 +158,24 @@ func storeImg(img io.Reader, title string, db *database.DB) (bson.ObjectId, bson
 	imgResized, err := resizeImg(img1, IMG_WIDTH_BIG)
 	if err != nil {
 		log.Error("Error resizing big image: ", err.Error())
-		return "", ""
+		return false
 	}
 	err = jpeg.Encode(fBig, imgResized, &jpgOptions)
 	if err != nil {
 		log.Error("Error encoding big image: ", err.Error())
-		return "", ""
+		return false
 	}
 	imgSmallResized, err := resizeImg(&img2, IMG_WIDTH_SMALL)
 	if err != nil {
 		log.Error("Error resizing small image: ", err.Error())
-		return "", ""
+		return false
 	}
 	err = jpeg.Encode(fSmall, imgSmallResized, &jpgOptions)
 	if err != nil {
 		log.Error("Error encoding small image: ", err.Error())
-		return "", ""
+		return false
 	}
-
+	return true
-	idBig, _ := fBig.Id().(bson.ObjectId)
-	idSmall, _ := fSmall.Id().(bson.ObjectId)
-	return idBig, idSmall
-}
-
-func createCoverFile(title string, db *database.DB) (*mgo.GridFile, error) {
-	fs := db.GetFS(FS_IMGS)
-	return fs.Create(title + ".jpg")
 }
 
 func resizeImg(imgReader io.Reader, width uint) (image.Image, error) {

database/books.go

@@ -1,7 +1,6 @@
 package database
 
 import (
-	"errors"
 	"gopkg.in/mgo.v2"
 	"gopkg.in/mgo.v2/bson"
 	"gopkgs.com/unidecode.v1"
@@ -14,7 +13,7 @@ const (
 )
 
 type Book struct {
-	Id          string `bson:"_id"`
+	Id          string
 	Title       string
 	Author      []string
 	Contributor string
@@ -31,10 +30,8 @@ type Book struct {
 	Coverage    string
 	Rights      string
 	Meta        string
-	File        bson.ObjectId
 	FileSize    int
-	Cover       bson.ObjectId
+	Cover       bool
-	CoverSmall  bson.ObjectId
 	Active      bool
 	Keywords    []string
 }
@@ -66,35 +63,41 @@ func _getBooks(coll *mgo.Collection, query bson.M, length int, start int) (books
 	}
 
 	err = q.All(&books)
-	for i, b := range books {
-		books[i].Id = bson.ObjectId(b.Id).Hex()
-	}
 	return
 }
 
 func getBookId(coll *mgo.Collection, id string) (Book, error) {
 	var book Book
-	if !bson.IsObjectIdHex(id) {
+	err := coll.Find(bson.M{"id": id}).One(&book)
-		return book, errors.New("Not valid book id")
-	}
-
-	err := coll.FindId(bson.ObjectIdHex(id)).One(&book)
-	book.Id = bson.ObjectId(book.Id).Hex()
 	return book, err
 }
 
 func deleteBook(coll *mgo.Collection, id string) error {
-	return coll.RemoveId(bson.ObjectIdHex(id))
+	return coll.Remove(bson.M{"id": id})
 }
 
 func updateBook(coll *mgo.Collection, id string, data map[string]interface{}) error {
-	data["keywords"] = keywords(data)
+	var book map[string]interface{}
-	return coll.UpdateId(bson.ObjectIdHex(id), bson.M{"$set": data})
+	err := coll.Find(bson.M{"id": id}).One(&book)
+	if err != nil {
+		return err
+	}
+	for k, v := range data {
+		book[k] = v
+	}
+
+	data["keywords"] = keywords(book)
+	return coll.Update(bson.M{"id": id}, bson.M{"$set": data})
 }
 
-func bookActive(coll *mgo.Collection, id string) bool {
+func activeBook(coll *mgo.Collection, id string) error {
+	data := map[string]interface{}{"active": true}
+	return coll.Update(bson.M{"id": id}, bson.M{"$set": data})
+}
+
+func isBookActive(coll *mgo.Collection, id string) bool {
 	var book Book
-	err := coll.FindId(bson.ObjectIdHex(id)).One(&book)
+	err := coll.Find(bson.M{"id": id}).One(&book)
 	if err != nil {
 		return false
 	}
@@ -123,15 +126,29 @@ func buildQuery(q string) bson.M {
 func keywords(b map[string]interface{}) (k []string) {
 	title, _ := b["title"].(string)
 	k = tokens(title)
-	author, _ := b["author"].([]string)
+
-	for _, a := range author {
+	k = append(k, listKeywords(b["author"])...)
-		k = append(k, tokens(a)...)
+
-	}
 	publisher, _ := b["publisher"].(string)
 	k = append(k, tokens(publisher)...)
-	subject, _ := b["subject"].([]string)
+
-	for _, s := range subject {
+	k = append(k, listKeywords(b["subject"])...)
-		k = append(k, tokens(s)...)
+	return
+}
+
+func listKeywords(v interface{}) (k []string) {
+	list, ok := v.([]string)
+	if !ok {
+		list, _ := v.([]interface{})
+		for _, e := range list {
+			str := e.(string)
+			k = append(k, tokens(str)...)
+		}
+		return
+	}
+
+	for _, e := range list {
+		k = append(k, tokens(e)...)
 	}
 	return
 }

database/books_test.go

@@ -5,6 +5,7 @@ import "testing"
 var book = map[string]interface{}{
 	"title":  "some title",
 	"author": []string{"Alice", "Bob"},
+	"id":     "r_m-IOzzIbA6QK5w",
 }
 
 func TestAddBook(t *testing.T) {
@@ -15,7 +16,7 @@ func TestAddBook(t *testing.T) {
 
 	books, num, err := db.GetNewBooks(1, 0)
 	if err != nil {
-		t.Fatalf("db.GetBooks() return an error: ", err)
+		t.Fatal("db.GetBooks() return an error: ", err)
 	}
 	if num < 1 {
 		t.Fatalf("db.GetBooks() didn't find any result.")
@@ -24,13 +25,61 @@ func TestAddBook(t *testing.T) {
 		t.Fatalf("db.GetBooks() didn't return any result.")
 	}
 	if books[0].Title != book["title"] {
-		t.Errorf("Book title don't match : '", books[0].Title, "' <=> '", book["title"], "'")
+		t.Error("Book title don't match : '", books[0].Title, "' <=> '", book["title"], "'")
+	}
+}
+
+func TestActiveBook(t *testing.T) {
+	db := Init(test_host, test_coll)
+	defer db.del()
+
+	tAddBook(t, db)
+	books, _, _ := db.GetNewBooks(1, 0)
+	id := books[0].Id
+
+	err := db.ActiveBook(id)
+	if err != nil {
+		t.Fatal("db.ActiveBook(", id, ") return an error: ", err)
+	}
+
+	b, err := db.GetBookId(id)
+	if err != nil {
+		t.Fatal("db.GetBookId(", id, ") return an error: ", err)
+	}
+	if b.Author[0] != books[0].Author[0] {
+		t.Error("Book author don't match : '", b.Author, "' <=> '", book["author"], "'")
+	}
+}
+
+func TestUpdateBookKeywords(t *testing.T) {
+	db := Init(test_host, test_coll)
+	defer db.del()
+
+	tAddBook(t, db)
+	books, _, _ := db.GetNewBooks(1, 0)
+
+	db.UpdateBook(books[0].Id, map[string]interface{}{"title": "Some other title"})
+	books, _, _ = db.GetNewBooks(1, 0)
+	keywords := books[0].Keywords
+
+	alice := false
+	bob := false
+	for _, e := range keywords {
+		if e == "alice" {
+			alice = true
+		}
+		if e == "bob" {
+			bob = true
+		}
+	}
+	if !alice || !bob {
+		t.Error("Alce or Bob are not in the keywords:", keywords)
 	}
 }
 
 func tAddBook(t *testing.T, db *DB) {
 	err := db.AddBook(book)
 	if err != nil {
-		t.Errorf("db.AddBook(", book, ") return an error: ", err)
+		t.Error("db.AddBook(", book, ") return an error: ", err)
 	}
 }

database/database.go

@@ -73,9 +73,14 @@ func (db *DB) UpdateBook(id string, data map[string]interface{}) error {
 	return updateBook(booksColl, id, data)
 }
 
-func (db *DB) BookActive(id string) bool {
+func (db *DB) ActiveBook(id string) error {
 	booksColl := db.session.DB(db.name).C(books_coll)
-	return bookActive(booksColl, id)
+	return activeBook(booksColl, id)
+}
+
+func (db *DB) IsBookActive(id string) bool {
+	booksColl := db.session.DB(db.name).C(books_coll)
+	return isBookActive(booksColl, id)
 }
 
 func (db *DB) User(name string) *User {
@@ -154,10 +159,6 @@ func (db *DB) UpdateDownloadedBooks() error {
 	return u.UpdateMostBooks("download")
 }
 
-func (db *DB) GetFS(prefix string) *mgo.GridFS {
-	return db.session.DB(db.name).GridFS(prefix)
-}
-
 func (db *DB) GetTags() ([]string, error) {
 	tagsColl := db.session.DB(db.name).C(tags_coll)
 	return GetTags(tagsColl)

reader.go

@@ -1,11 +1,15 @@
 package main
 
+import log "github.com/cihub/seelog"
+
 import (
+	"bytes"
 	"git.gitorious.org/go-pkg/epubgo.git"
 	"git.gitorious.org/trantor/trantor.git/database"
+	"git.gitorious.org/trantor/trantor.git/storage"
 	"github.com/gorilla/mux"
 	"io"
-	"labix.org/v2/mgo/bson"
+	"io/ioutil"
 	"net/http"
 	"strconv"
 	"strings"
@@ -134,6 +138,7 @@ func readStartHandler(h handler) {
 	id := mux.Vars(h.r)["id"]
 	e, _ := openReadEpub(h)
 	if e == nil {
+		log.Warn("Open epub returns an empty file")
 		notFound(h)
 		return
 	}
@@ -141,6 +146,7 @@ func readStartHandler(h handler) {
 
 	it, err := e.Spine()
 	if err != nil {
+		log.Warn("No spine in the epub")
 		notFound(h)
 		return
 	}
@@ -175,7 +181,7 @@ func readHandler(h handler) {
 func openReadEpub(h handler) (*epubgo.Epub, database.Book) {
 	var book database.Book
 	id := mux.Vars(h.r)["id"]
-	if !bson.IsObjectIdHex(id) {
+	if id == "" {
 		return nil, book
 	}
 	book, err := h.db.GetBookId(id)
@@ -188,7 +194,7 @@ func openReadEpub(h handler) (*epubgo.Epub, database.Book) {
 			return nil, book
 		}
 	}
-	e, err := OpenBook(book.File, h.db)
+	e, err := openBook(book.Id, h.store)
 	if err != nil {
 		return nil, book
 	}
@@ -199,7 +205,7 @@ func contentHandler(h handler) {
 	vars := mux.Vars(h.r)
 	id := vars["id"]
 	file := vars["file"]
-	if file == "" || !bson.IsObjectIdHex(id) {
+	if file == "" || id == "" {
 		notFound(h)
 		return
 	}
@@ -215,7 +221,7 @@ func contentHandler(h handler) {
 			return
 		}
 	}
-	e, err := OpenBook(book.File, h.db)
+	e, err := openBook(book.Id, h.store)
 	if err != nil {
 		notFound(h)
 		return
@@ -230,3 +236,16 @@ func contentHandler(h handler) {
 	defer html.Close()
 	io.Copy(h.w, html)
 }
+
+func openBook(id string, store *storage.Store) (*epubgo.Epub, error) {
+	f, err := store.Get(id, EPUB_FILE)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	buff, err := ioutil.ReadAll(f)
+	reader := bytes.NewReader(buff)
+
+	return epubgo.Load(reader, int64(len(buff)))
+}

stats.go

@@ -4,32 +4,49 @@ import log "github.com/cihub/seelog"
 
 import (
 	"git.gitorious.org/trantor/trantor.git/database"
+	"git.gitorious.org/trantor/trantor.git/storage"
 	"github.com/gorilla/mux"
-	"labix.org/v2/mgo/bson"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 )
 
+const (
+	stats_version = 2
+)
+
 type handler struct {
-	w    http.ResponseWriter
+	w     http.ResponseWriter
-	r    *http.Request
+	r     *http.Request
-	sess *Session
+	sess  *Session
-	db   *database.DB
+	db    *database.DB
+	store *storage.Store
 }
 
-func InitStats(database *database.DB) {
+type StatsGatherer struct {
-	statsChannel = make(chan statsRequest, CHAN_SIZE)
+	db      *database.DB
-	go statsWorker(database)
+	store   *storage.Store
+	channel chan statsRequest
 }
 
-func GatherStats(function func(handler), database *database.DB) func(http.ResponseWriter, *http.Request) {
+func InitStats(database *database.DB, store *storage.Store) *StatsGatherer {
+	sg := new(StatsGatherer)
+	sg.channel = make(chan statsRequest, CHAN_SIZE)
+	sg.db = database
+	sg.store = store
+
+	go sg.worker()
+	return sg
+}
+
+func (sg StatsGatherer) Gather(function func(handler)) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		log.Info("Query ", r.Method, " ", r.RequestURI)
 
 		var h handler
-		h.db = database.Copy()
+		h.store = sg.store
+		h.db = sg.db.Copy()
 		defer h.db.Close()
 
 		h.w = w
@@ -37,12 +54,10 @@ func GatherStats(function func(handler), database *database.DB) func(http.Respon
 		h.sess = GetSession(r, h.db)
 		function(h)
 
-		statsChannel <- statsRequest{bson.Now(), mux.Vars(r), h.sess, r}
+		sg.channel <- statsRequest{time.Now(), mux.Vars(r), h.sess, r}
 	}
 }
 
-var statsChannel chan statsRequest
-
 type statsRequest struct {
 	date time.Time
 	vars map[string]string
@@ -50,16 +65,17 @@ type statsRequest struct {
 	r    *http.Request
 }
 
-func statsWorker(database *database.DB) {
+func (sg StatsGatherer) worker() {
-	db := database.Copy()
+	db := sg.db.Copy()
 	defer db.Close()
 
-	for req := range statsChannel {
+	for req := range sg.channel {
 		stats := make(map[string]interface{})
 		appendFiles(req.r, stats)
 		appendMuxVars(req.vars, stats)
 		appendUrl(req.r, stats)
 		appendSession(req.sess, stats)
+		stats["version"] = stats_version
 		stats["method"] = req.r.Method
 		stats["date"] = req.date
 		db.AddStats(stats)
@@ -141,16 +157,12 @@ func appendMuxVars(vars map[string]string, stats map[string]interface{}) {
 	for key, value := range vars {
 		switch {
 		case key == "id":
-			if bson.IsObjectIdHex(value) {
+			stats["id"] = value
-				stats["id"] = bson.ObjectIdHex(value)
-			}
 		case key == "ids":
-			var objectIds []bson.ObjectId
+			var objectIds []string
 			ids := strings.Split(value, "/")
 			for _, id := range ids {
-				if bson.IsObjectIdHex(value) {
+				objectIds = append(objectIds, id)
-					objectIds = append(objectIds, bson.ObjectIdHex(id))
-				}
 			}
 			if len(objectIds) > 0 {
 				stats["ids"] = objectIds

storage/dir.go

@@ -0,0 +1,42 @@
+package storage
+
+import p "path"
+
+import (
+	"os"
+)
+
+const (
+	dir_depth = 2
+	encodeURL = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
+)
+
+func mkstore(path string) error {
+	return _mkstore(path, dir_depth)
+}
+
+func _mkstore(path string, depth int) error {
+	err := os.MkdirAll(path, os.ModePerm)
+	if err != nil || depth == 0 {
+		return err
+	}
+
+	for _, l := range encodeURL {
+		next_path := p.Join(path, string(l))
+		err = _mkstore(next_path, depth-1)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func idPath(storePath string, id string) string {
+	path := storePath
+	for i := 0; i < dir_depth; i++ {
+		dir := string(id[i])
+		path = p.Join(path, dir)
+	}
+	path = p.Join(path, id)
+	return path
+}

storage/storage.go

@@ -0,0 +1,57 @@
+package storage
+
+import p "path"
+
+import (
+	"io"
+	"os"
+)
+
+type Store struct {
+	path string
+}
+
+func Init(path string) (*Store, error) {
+	st := new(Store)
+	st.path = path
+
+	_, err := os.Stat(path)
+	if err != nil {
+		err = mkstore(st.path)
+	}
+	return st, err
+}
+
+func (st *Store) Create(id string, name string) (io.WriteCloser, error) {
+	path := idPath(st.path, id)
+	err := os.MkdirAll(path, os.ModePerm)
+	if err != nil {
+		return nil, err
+	}
+
+	return os.Create(p.Join(path, name))
+}
+
+func (st *Store) Store(id string, file io.Reader, name string) (size int64, err error) {
+	dest, err := st.Create(id, name)
+	if err != nil {
+		return 0, err
+	}
+	defer dest.Close()
+
+	return io.Copy(dest, file)
+}
+
+func (st *Store) Get(id string, name string) (io.ReadCloser, error) {
+	path := idPath(st.path, id)
+	return os.Open(p.Join(path, name))
+}
+
+func (st *Store) Delete(id string) error {
+	path := idPath(st.path, id)
+	return os.RemoveAll(path)
+}
+
+func (st *Store) del() {
+	os.RemoveAll(st.path)
+}

storage/storage_test.go

@@ -0,0 +1,113 @@
+package storage
+
+import "testing"
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+)
+
+const (
+	test_path = "/tmp/store"
+	test_book = `
+		HARI SELDON-… born in the 11,988th year of the Galactic Era; died
+		12,069. The dates are more commonly given in terms of the current
+		Foundational Era as - 79 to the year 1 F.E. Born to middle-class
+		parents on Helicon, Arcturus sector (where his father, in a legend of
+		doubtful authenticity, was a tobacco grower in the hydroponic plants
+		of the planet), he early showed amazing ability in mathematics.
+		Anecdotes concerning his ability are innumerable, and some are 
+		contradictory. At the age of two, he is said to have …`
+	test_id = "1234567890abcdef"
+)
+
+func TestInit(t *testing.T) {
+	st, err := Init(test_path)
+	if err != nil {
+		t.Fatal("An error ocurred initializing the store =>", err)
+	}
+	defer st.del()
+
+	info, err := os.Stat(test_path)
+	if err != nil {
+		t.Fatal("An error ocurred =>", err)
+	}
+	if !info.Mode().IsDir() {
+		t.Errorf(test_path, " is not dir.")
+	}
+
+	info, err = os.Stat(test_path + "/a/M")
+	if err != nil {
+		t.Fatal("An error ocurred =>", err)
+	}
+	if !info.Mode().IsDir() {
+		t.Errorf(test_path, " is not dir.")
+	}
+}
+
+func TestStore(t *testing.T) {
+	st, err := Init(test_path)
+	defer st.del()
+
+	_, err = st.Store(test_id, strings.NewReader(test_book), "epub")
+	if err != nil {
+		t.Fatal("An error ocurred storing the book =>", err)
+	}
+	book, err := st.Get(test_id, "epub")
+	if err != nil {
+		t.Fatal("An error ocurred getting the book =>", err)
+	}
+
+	content, err := ioutil.ReadAll(book)
+	if err != nil {
+		t.Fatal("An error ocurred reading the book =>", err)
+	}
+	if !bytes.Equal(content, []byte(test_book)) {
+		t.Error("Not the same content")
+	}
+}
+
+func TestCreate(t *testing.T) {
+	st, err := Init(test_path)
+	defer st.del()
+
+	f, err := st.Create(test_id, "img")
+	if err != nil {
+		t.Fatal("An error ocurred storing the book =>", err)
+	}
+	io.Copy(f, strings.NewReader(test_book))
+	img, err := st.Get(test_id, "img")
+	if err != nil {
+		t.Fatal("An error ocurred getting the book =>", err)
+	}
+
+	content, err := ioutil.ReadAll(img)
+	if err != nil {
+		t.Fatal("An error ocurred reading the book =>", err)
+	}
+	if !bytes.Equal(content, []byte(test_book)) {
+		t.Error("Not the same content")
+	}
+}
+
+func TestDelete(t *testing.T) {
+	st, err := Init(test_path)
+	defer st.del()
+
+	_, err = st.Store(test_id, strings.NewReader(test_book), "epub")
+	if err != nil {
+		t.Fatal("An error ocurred storing the book =>", err)
+	}
+	err = st.Delete(test_id)
+	if err != nil {
+		t.Fatal("An error ocurred deleteing id =>", err)
+	}
+
+	_, err = st.Get(test_id, "epub")
+	if err == nil {
+		t.Fatal("Retrieve book without error.")
+	}
+}

store.go

@@ -1,115 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"git.gitorious.org/go-pkg/epubgo.git"
-	"git.gitorious.org/trantor/trantor.git/database"
-	"io"
-	"io/ioutil"
-	"labix.org/v2/mgo/bson"
-	"regexp"
-	"strings"
-)
-
-func OpenBook(id bson.ObjectId, db *database.DB) (*epubgo.Epub, error) {
-	fs := db.GetFS(FS_BOOKS)
-	f, err := fs.OpenId(id)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	buff, err := ioutil.ReadAll(f)
-	reader := bytes.NewReader(buff)
-
-	return epubgo.Load(reader, int64(len(buff)))
-}
-
-func StoreNewFile(name string, file io.Reader, db *database.DB) (bson.ObjectId, int64, error) {
-	fs := db.GetFS(FS_BOOKS)
-	fw, err := fs.Create(name)
-	if err != nil {
-		return "", 0, err
-	}
-	defer fw.Close()
-
-	size, err := io.Copy(fw, file)
-	id, _ := fw.Id().(bson.ObjectId)
-	return id, size, err
-}
-
-func DeleteFile(id bson.ObjectId, db *database.DB) error {
-	fs := db.GetFS(FS_BOOKS)
-	return fs.RemoveId(id)
-}
-
-func DeleteCover(id bson.ObjectId, db *database.DB) error {
-	fs := db.GetFS(FS_IMGS)
-	return fs.RemoveId(id)
-}
-
-func DeleteBook(book database.Book, db *database.DB) {
-	if book.Cover != "" {
-		DeleteCover(book.Cover, db)
-	}
-	if book.CoverSmall != "" {
-		DeleteCover(book.CoverSmall, db)
-	}
-	DeleteFile(book.File, db)
-}
-
-func cleanStr(str string) string {
-	str = strings.Replace(str, "'", "'", -1)
-	exp, _ := regexp.Compile("&[^;]*;")
-	str = exp.ReplaceAllString(str, "")
-	exp, _ = regexp.Compile("[ ,]*$")
-	str = exp.ReplaceAllString(str, "")
-	return str
-}
-
-func parseAuthr(creator []string) []string {
-	exp1, _ := regexp.Compile("^(.*\\( *([^\\)]*) *\\))*$")
-	exp2, _ := regexp.Compile("^[^:]*: *(.*)$")
-	res := make([]string, len(creator))
-	for i, s := range creator {
-		auth := exp1.FindStringSubmatch(s)
-		if auth != nil {
-			res[i] = cleanStr(strings.Join(auth[2:], ", "))
-		} else {
-			auth := exp2.FindStringSubmatch(s)
-			if auth != nil {
-				res[i] = cleanStr(auth[1])
-			} else {
-				res[i] = cleanStr(s)
-			}
-		}
-	}
-	return res
-}
-
-func parseDescription(description []string) string {
-	str := cleanStr(strings.Join(description, "\n"))
-	str = strings.Replace(str, "</p>", "\n", -1)
-	exp, _ := regexp.Compile("<[^>]*>")
-	str = exp.ReplaceAllString(str, "")
-	str = strings.Replace(str, "&amp;", "&", -1)
-	str = strings.Replace(str, "&lt;", "<", -1)
-	str = strings.Replace(str, "&gt;", ">", -1)
-	str = strings.Replace(str, "\\n", "\n", -1)
-	return str
-}
-
-func parseSubject(subject []string) []string {
-	var res []string
-	for _, s := range subject {
-		res = append(res, strings.Split(s, " / ")...)
-	}
-	return res
-}
-
-func parseDate(date []string) string {
-	if len(date) == 0 {
-		return ""
-	}
-	return strings.Replace(date[0], "Unspecified: ", "", -1)
-}

templates/index.html

@@ -26,7 +26,7 @@
 	<li class="span2">
 		<div class="thumbnail centered" style="border:none;">
 		<a href="/book/{{.Id}}">
-		{{if .CoverSmall}}<div class="down"><img  class="img-rounded" src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" /></div>{{end}}
+		{{if .Cover}}<div class="down"><img  class="img-rounded" src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" /></div>{{end}}
 		<p><strong>{{.Title}}</strong></p>
 		</a>
 		</div>
@@ -44,7 +44,7 @@
 	<li class="span2">
 		<div class="thumbnail centered" style="border:none;">
 		<a href="/book/{{.Id}}">
-		{{if .CoverSmall}}<div class="down"><img  class="img-rounded" src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" /></div>{{end}}
+		{{if .Cover}}<div class="down"><img  class="img-rounded" src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" /></div>{{end}}
 		<p><strong>{{.Title}}</strong></p>
 		</a>
 		</div>
@@ -62,7 +62,7 @@
 	<li class="span2">
 		<div class="thumbnail centered" style="border:none;">
 		<a href="/book/{{.Id}}">
-		{{if .CoverSmall}}<div class="down"><img  class="img-rounded" src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" /></div>{{end}}
+		{{if .Cover}}<div class="down"><img  class="img-rounded" src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" /></div>{{end}}
 		<p><strong>{{.Title}}</strong></p>
 		</a>
 		</div>

templates/new.html

@@ -26,7 +26,7 @@
 	{{with .B}}
 	<div class="row">
 		<div class="span1">
-		<p class="pull-right">{{if .CoverSmall}}<img src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" />{{end}}</p>
+		<p class="pull-right">{{if .Cover}}<img src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" />{{end}}</p>
 		</div>
 		<div class="span9">
 		<p><a href="/search/?q=title:{{.Title}}"><strong>{{.Title}}</strong></a> <small>({{$titleFound}})</small><br />

templates/search.html

@@ -21,7 +21,7 @@
 	{{range .}}
 	<div class="row">
 		<div class="span1">
-		<p class="pull-right"><a href="/book/{{.Id}}">{{if .CoverSmall}}<img class="img-rounded" src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" />{{end}}</a></p>
+		<p class="pull-right"><a href="/book/{{.Id}}">{{if .Cover}}<img class="img-rounded" src="/cover/{{.Id}}/small/{{.Title}}.jpg" alt="{{.Title}}" />{{end}}</a></p>
 		</div>
 		<div class="span10 well">
 		<div class="row">

trantor.go

@@ -4,9 +4,11 @@ import log "github.com/cihub/seelog"
 
 import (
 	"git.gitorious.org/trantor/trantor.git/database"
+	"git.gitorious.org/trantor/trantor.git/storage"
 	"github.com/gorilla/mux"
 	"io"
 	"net/http"
+	"os"
 	"strings"
 )
 
@@ -71,8 +73,7 @@ func downloadHandler(h handler) {
 		}
 	}
 
-	fs := h.db.GetFS(FS_BOOKS)
+	f, err := h.store.Get(book.Id, EPUB_FILE)
-	f, err := fs.OpenId(book.File)
 	if err != nil {
 		notFound(h)
 		return
@@ -81,7 +82,7 @@ func downloadHandler(h handler) {
 
 	headers := h.w.Header()
 	headers["Content-Type"] = []string{"application/epub+zip"}
-	headers["Content-Disposition"] = []string{"attachment; filename=\"" + f.Name() + "\""}
+	headers["Content-Disposition"] = []string{"attachment; filename=\"" + book.Title + ".epub\""}
 
 	io.Copy(h.w, f)
 }
@@ -137,47 +138,55 @@ func main() {
 	db := database.Init(DB_IP, DB_NAME)
 	defer db.Close()
 
-	InitTasks(db)
+	store, err := storage.Init(STORE_PATH)
-	InitStats(db)
+	if err != nil {
-	InitUpload(db)
+		log.Critical("Problem initializing store: ", err)
+		os.Exit(1)
+	}
 
-	initRouter(db)
+	InitTasks(db)
+	sg := InitStats(db, store)
+	InitUpload(db, store)
+
+	initRouter(db, sg)
 	log.Error(http.ListenAndServe(":"+PORT, nil))
 }
 
-func initRouter(db *database.DB) {
+func initRouter(db *database.DB, sg *StatsGatherer) {
+	const id_pattern = "[0-9a-zA-Z\\-\\_]{16}"
+
 	r := mux.NewRouter()
 	var notFoundHandler http.HandlerFunc
-	notFoundHandler = GatherStats(notFound, db)
+	notFoundHandler = sg.Gather(notFound)
 	r.NotFoundHandler = notFoundHandler
 
-	r.HandleFunc("/", GatherStats(indexHandler, db))
+	r.HandleFunc("/", sg.Gather(indexHandler))
-	r.HandleFunc("/book/{id:[0-9a-fA-F]+}", GatherStats(bookHandler, db))
+	r.HandleFunc("/book/{id:"+id_pattern+"}", sg.Gather(bookHandler))
-	r.HandleFunc("/search/", GatherStats(searchHandler, db))
+	r.HandleFunc("/search/", sg.Gather(searchHandler))
-	r.HandleFunc("/upload/", GatherStats(uploadHandler, db)).Methods("GET")
+	r.HandleFunc("/upload/", sg.Gather(uploadHandler)).Methods("GET")
-	r.HandleFunc("/upload/", GatherStats(uploadPostHandler, db)).Methods("POST")
+	r.HandleFunc("/upload/", sg.Gather(uploadPostHandler)).Methods("POST")
-	r.HandleFunc("/login/", GatherStats(loginHandler, db)).Methods("GET")
+	r.HandleFunc("/login/", sg.Gather(loginHandler)).Methods("GET")
-	r.HandleFunc("/login/", GatherStats(loginPostHandler, db)).Methods("POST")
+	r.HandleFunc("/login/", sg.Gather(loginPostHandler)).Methods("POST")
-	r.HandleFunc("/create_user/", GatherStats(createUserHandler, db)).Methods("POST")
+	r.HandleFunc("/create_user/", sg.Gather(createUserHandler)).Methods("POST")
-	r.HandleFunc("/logout/", GatherStats(logoutHandler, db))
+	r.HandleFunc("/logout/", sg.Gather(logoutHandler))
-	r.HandleFunc("/new/", GatherStats(newHandler, db))
+	r.HandleFunc("/new/", sg.Gather(newHandler))
-	r.HandleFunc("/store/{ids:([0-9a-fA-F]+/)+}", GatherStats(storeHandler, db))
+	r.HandleFunc("/store/{ids:("+id_pattern+"/)+}", sg.Gather(storeHandler))
-	r.HandleFunc("/delete/{ids:([0-9a-fA-F]+/)+}", GatherStats(deleteHandler, db))
+	r.HandleFunc("/delete/{ids:("+id_pattern+"/)+}", sg.Gather(deleteHandler))
-	r.HandleFunc("/read/{id:[0-9a-fA-F]+}", GatherStats(readStartHandler, db))
+	r.HandleFunc("/read/{id:"+id_pattern+"}", sg.Gather(readStartHandler))
-	r.HandleFunc("/read/{id:[0-9a-fA-F]+}/{file:.*}", GatherStats(readHandler, db))
+	r.HandleFunc("/read/{id:"+id_pattern+"}/{file:.*}", sg.Gather(readHandler))
-	r.HandleFunc("/content/{id:[0-9a-fA-F]+}/{file:.*}", GatherStats(contentHandler, db))
+	r.HandleFunc("/content/{id:"+id_pattern+"}/{file:.*}", sg.Gather(contentHandler))
-	r.HandleFunc("/edit/{id:[0-9a-fA-F]+}", GatherStats(editHandler, db))
+	r.HandleFunc("/edit/{id:"+id_pattern+"}", sg.Gather(editHandler))
-	r.HandleFunc("/save/{id:[0-9a-fA-F]+}", GatherStats(saveHandler, db)).Methods("POST")
+	r.HandleFunc("/save/{id:"+id_pattern+"}", sg.Gather(saveHandler)).Methods("POST")
-	r.HandleFunc("/about/", GatherStats(aboutHandler, db))
+	r.HandleFunc("/about/", sg.Gather(aboutHandler))
-	r.HandleFunc("/help/", GatherStats(helpHandler, db))
+	r.HandleFunc("/help/", sg.Gather(helpHandler))
-	r.HandleFunc("/download/{id:[0-9a-fA-F]+}/{epub:.*}", GatherStats(downloadHandler, db))
+	r.HandleFunc("/download/{id:"+id_pattern+"}/{epub:.*}", sg.Gather(downloadHandler))
-	r.HandleFunc("/cover/{id:[0-9a-fA-F]+}/{size}/{img:.*}", GatherStats(coverHandler, db))
+	r.HandleFunc("/cover/{id:"+id_pattern+"}/{size}/{img:.*}", sg.Gather(coverHandler))
-	r.HandleFunc("/dashboard/", GatherStats(dashboardHandler, db))
+	r.HandleFunc("/dashboard/", sg.Gather(dashboardHandler))
-	r.HandleFunc("/settings/", GatherStats(settingsHandler, db))
+	r.HandleFunc("/settings/", sg.Gather(settingsHandler))
-	r.HandleFunc("/stats/", GatherStats(statsHandler, db))
+	r.HandleFunc("/stats/", sg.Gather(statsHandler))
-	r.HandleFunc("/news/", GatherStats(newsHandler, db))
+	r.HandleFunc("/news/", sg.Gather(newsHandler))
-	r.HandleFunc("/news/edit", GatherStats(editNewsHandler, db)).Methods("GET")
+	r.HandleFunc("/news/edit", sg.Gather(editNewsHandler)).Methods("GET")
-	r.HandleFunc("/news/edit", GatherStats(postNewsHandler, db)).Methods("POST")
+	r.HandleFunc("/news/edit", sg.Gather(postNewsHandler)).Methods("POST")
 	h := http.FileServer(http.Dir(IMG_PATH))
 	r.Handle("/img/{img}", http.StripPrefix("/img/", h))
 	h = http.FileServer(http.Dir(CSS_PATH))

upload.go

@@ -4,16 +4,20 @@ import log "github.com/cihub/seelog"
 
 import (
 	"bytes"
+	"crypto/rand"
+	"encoding/base64"
 	"git.gitorious.org/go-pkg/epubgo.git"
 	"git.gitorious.org/trantor/trantor.git/database"
+	"git.gitorious.org/trantor/trantor.git/storage"
 	"io/ioutil"
 	"mime/multipart"
+	"regexp"
 	"strings"
 )
 
-func InitUpload(database *database.DB) {
+func InitUpload(database *database.DB, store *storage.Store) {
 	uploadChannel = make(chan uploadRequest, CHAN_SIZE)
-	go uploadWorker(database)
+	go uploadWorker(database, store)
 }
 
 var uploadChannel chan uploadRequest
@@ -23,16 +27,16 @@ type uploadRequest struct {
 	filename string
 }
 
-func uploadWorker(database *database.DB) {
+func uploadWorker(database *database.DB, store *storage.Store) {
 	db := database.Copy()
 	defer db.Close()
 
 	for req := range uploadChannel {
-		processFile(req, db)
+		processFile(req, db, store)
 	}
 }
 
-func processFile(req uploadRequest, db *database.DB) {
+func processFile(req uploadRequest, db *database.DB, store *storage.Store) {
 	defer req.file.Close()
 
 	epub, err := openMultipartEpub(req.file)
@@ -42,20 +46,18 @@ func processFile(req uploadRequest, db *database.DB) {
 	}
 	defer epub.Close()
 
-	book := parseFile(epub, db)
+	book, id := parseFile(epub, store)
-	title, _ := book["title"].(string)
 	req.file.Seek(0, 0)
-	id, size, err := StoreNewFile(title+".epub", req.file, db)
+	size, err := store.Store(id, req.file, EPUB_FILE)
 	if err != nil {
-		log.Error("Error storing book (", title, "): ", err)
+		log.Error("Error storing book (", id, "): ", err)
 		return
 	}
 
-	book["file"] = id
 	book["filesize"] = size
 	err = db.AddBook(book)
 	if err != nil {
-		log.Error("Error storing metadata (", title, "): ", err)
+		log.Error("Error storing metadata (", id, "): ", err)
 		return
 	}
 	log.Info("File uploaded: ", req.filename)
@@ -104,7 +106,7 @@ func openMultipartEpub(file multipart.File) (*epubgo.Epub, error) {
 	return epubgo.Load(reader, int64(len(buff)))
 }
 
-func parseFile(epub *epubgo.Epub, db *database.DB) map[string]interface{} {
+func parseFile(epub *epubgo.Epub, store *storage.Store) (metadata map[string]interface{}, id string) {
 	book := map[string]interface{}{}
 	for _, m := range epub.MetadataFields() {
 		data, err := epub.Metadata(m)
@@ -135,12 +137,71 @@ func parseFile(epub *epubgo.Epub, db *database.DB) map[string]interface{} {
 			book[m] = strings.Join(data, ", ")
 		}
 	}
-	title, _ := book["title"].(string)
+
-	book["file"] = nil
+	id = genId()
-	cover, coverSmall := GetCover(epub, title, db)
+	book["id"] = id //TODO
-	if cover != "" {
+	book["cover"] = GetCover(epub, id, store)
-		book["cover"] = cover
+	return book, id
-		book["coversmall"] = coverSmall
+}
-	}
+
-	return book
+func genId() string {
+	b := make([]byte, 12)
+	rand.Read(b)
+	return base64.URLEncoding.EncodeToString(b)
+}
+
+func cleanStr(str string) string {
+	str = strings.Replace(str, "'", "'", -1)
+	exp, _ := regexp.Compile("&[^;]*;")
+	str = exp.ReplaceAllString(str, "")
+	exp, _ = regexp.Compile("[ ,]*$")
+	str = exp.ReplaceAllString(str, "")
+	return str
+}
+
+func parseAuthr(creator []string) []string {
+	exp1, _ := regexp.Compile("^(.*\\( *([^\\)]*) *\\))*$")
+	exp2, _ := regexp.Compile("^[^:]*: *(.*)$")
+	res := make([]string, len(creator))
+	for i, s := range creator {
+		auth := exp1.FindStringSubmatch(s)
+		if auth != nil {
+			res[i] = cleanStr(strings.Join(auth[2:], ", "))
+		} else {
+			auth := exp2.FindStringSubmatch(s)
+			if auth != nil {
+				res[i] = cleanStr(auth[1])
+			} else {
+				res[i] = cleanStr(s)
+			}
+		}
+	}
+	return res
+}
+
+func parseDescription(description []string) string {
+	str := cleanStr(strings.Join(description, "\n"))
+	str = strings.Replace(str, "</p>", "\n", -1)
+	exp, _ := regexp.Compile("<[^>]*>")
+	str = exp.ReplaceAllString(str, "")
+	str = strings.Replace(str, "&amp;", "&", -1)
+	str = strings.Replace(str, "&lt;", "<", -1)
+	str = strings.Replace(str, "&gt;", ">", -1)
+	str = strings.Replace(str, "\\n", "\n", -1)
+	return str
+}
+
+func parseSubject(subject []string) []string {
+	var res []string
+	for _, s := range subject {
+		res = append(res, strings.Split(s, " / ")...)
+	}
+	return res
+}
+
+func parseDate(date []string) string {
+	if len(date) == 0 {
+		return ""
+	}
+	return strings.Replace(date[0], "Unspecified: ", "", -1)
 }