archives/trantor
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improve memory footprint on upload books

Browse source
This commit is contained in:
Las Zenow 2017-09-26 23:46:03 +00:00
parent 04b636a62c
commit 548bcd3b70
2 changed files with 51 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
lib/stats.go
View file

@ -108,6 +108,11 @@ type statsRequest struct {
func (sg StatsGatherer) worker() { func (sg StatsGatherer) worker() {
for req := range sg.channel { for req := range sg.channel {
sg.save(req)
}
}
func (sg StatsGatherer) save(req statsRequest) {
var err error var err error
id := mux.Vars(req.r)["id"] id := mux.Vars(req.r)["id"]
search := strings.Join(req.r.Form["q"], " ") search := strings.Join(req.r.Form["q"], " ")
@ -139,4 +144,3 @@ func (sg StatsGatherer) worker() {
log.Warn("Problem incrementing visits: ", err) log.Warn("Problem incrementing visits: ", err)
} }
} }
}

32
lib/upload.go
View file

@ -3,10 +3,8 @@ package trantor
import ( import (
log "github.com/cihub/seelog" log "github.com/cihub/seelog"
"bytes"
"crypto/rand" "crypto/rand"
"encoding/base64" "encoding/base64"
"io/ioutil"
"mime/multipart" "mime/multipart"
"github.com/gorilla/mux" "github.com/gorilla/mux"
@ -29,24 +27,23 @@ var uploadChannel chan uploadRequest
type uploadRequest struct { type uploadRequest struct {
file multipart.File file multipart.File
filename string header *multipart.FileHeader
id int id int
} }
func uploadWorker(database database.DB, store storage.Store) { func uploadWorker(database database.DB, store storage.Store) {
for req := range uploadChannel { for req := range uploadChannel {
processFile(req, database, store) req.processFile(database, store)
} }
} }
func processFile(req uploadRequest, db database.DB, store storage.Store) { func (req uploadRequest) processFile(db database.DB, store storage.Store) {
defer req.file.Close() defer req.file.Close()
epub, err := openMultipartEpub(req.file) epub, err := epubgo.Load(req.file, req.header.Size)
if err != nil { if err != nil {
log.Warn("Not valid epub uploaded file ", req.filename, ": ", err) log.Warn("Not valid epub uploaded file ", req.header.Filename, ": ", err)
db.UpdateSubmission(req.id, "It is not a valid epub file", nil) db.UpdateSubmission(req.id, "It is not a valid epub file", nil)
return
} }
defer epub.Close() defer epub.Close()
@ -70,18 +67,25 @@ func processFile(req uploadRequest, db database.DB, store storage.Store) {
db.UpdateSubmission(req.id, "There was a problem in our server", nil) db.UpdateSubmission(req.id, "There was a problem in our server", nil)
return return
} }
log.Info("File uploaded: ", req.filename) log.Info("File uploaded: ", req.header.Filename)
db.UpdateSubmission(req.id, "Waiting for moderation", &book) db.UpdateSubmission(req.id, "Waiting for moderation", &book)
} }
func uploadPostHandler(h handler) { func uploadPostHandler(h handler) {
const _2M int64 = (1 << 20) * 2
if h.ro { if h.ro {
h.sess.Notify("Upload failed!", "The library is in Read Only mode, no books can be uploaded", "error") h.sess.Notify("Upload failed!", "The library is in Read Only mode, no books can be uploaded", "error")
uploadHandler(h) uploadHandler(h)
return return
} }
h.r.ParseMultipartForm(20000000) if err := h.r.ParseMultipartForm(_2M); nil != err {
log.Error("Can't parse form: ", err)
return
}
defer h.r.MultipartForm.RemoveAll()
filesForm := h.r.MultipartForm.File["epub"] filesForm := h.r.MultipartForm.File["epub"]
submissionID := genID() submissionID := genID()
for _, f := range filesForm { for _, f := range filesForm {
@ -104,7 +108,7 @@ func uploadPostHandler(h handler) {
if err != nil { if err != nil {
log.Error("Can add submission to db for ", f.Filename, ": ", err) log.Error("Can add submission to db for ", f.Filename, ": ", err)
} }
uploadChannel <- uploadRequest{file, f.Filename, id} uploadChannel <- uploadRequest{file, f, id}
} }
_uploadHandler(h, submissionID) _uploadHandler(h, submissionID)
} }
@ -146,12 +150,6 @@ type submissionData struct {
Submissions []database.Submission Submissions []database.Submission
} }
func openMultipartEpub(file multipart.File) (*epubgo.Epub, error) {
buff, _ := ioutil.ReadAll(file)
reader := bytes.NewReader(buff)
return epubgo.Load(reader, int64(len(buff)))
}
func genID() string { func genID() string {
b := make([]byte, 12) b := make([]byte, 12)
rand.Read(b) rand.Read(b)