Describe the bug
The upload performance when using the drivers (mem, file, s3 the only one tested) is very slow compared to direct use of native/alternative package.
To Reproduce
We will upload a large file ~4.5GB via curl
curl -X POST -F "file=@large.bin" http://localhost:8080/upload
Consider this file upload to local file:
package main
import (
"fmt"
"io"
"net/http"
"os"
"path/filepath"
)
func uploadHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
return
}
// Parse multipart form with 100 MB max memory (rest stored on disk)
err := r.ParseMultipartForm(100 << 20)
if err != nil {
http.Error(w, "Failed to parse multipart form: "+err.Error(), http.StatusBadRequest)
return
}
file, header, err := r.FormFile("file")
if err != nil {
http.Error(w, "Failed to get file from form: "+err.Error(), http.StatusBadRequest)
return
}
defer file.Close()
// Create destination file (save in current dir)
dstPath := filepath.Join(".", header.Filename)
dst, err := os.Create(dstPath)
if err != nil {
http.Error(w, "Failed to create file: "+err.Error(), http.StatusInternalServerError)
return
}
defer dst.Close()
// Copy uploaded file to disk
written, err := io.Copy(dst, file)
if err != nil {
http.Error(w, "Failed to save file: "+err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
fmt.Fprintf(w, "Uploaded %s (%d bytes) successfully\n", header.Filename, written)
}
func main() {
http.HandleFunc("/upload", uploadHandler)
fmt.Println("Listening on :8080...")
err := http.ListenAndServe(":8080", nil)
if err != nil {
panic(err)
}
}
On my machine (MacBook Pro M1 Max) it takes about ~8 sec e2e.
The equivalent using blob
package main
import (
"context"
"fmt"
"io"
"net/http"
"path/filepath"
"gocloud.dev/blob"
_ "gocloud.dev/blob/fileblob"
)
func uploadHandler(bucket *blob.Bucket) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
return
}
err := r.ParseMultipartForm(100 << 20)
if err != nil {
http.Error(w, "Failed to parse multipart form: "+err.Error(), http.StatusBadRequest)
return
}
file, header, err := r.FormFile("file")
if err != nil {
http.Error(w, "Failed to get file from form: "+err.Error(), http.StatusBadRequest)
return
}
defer file.Close()
ctx := r.Context()
// Use filename as blob key
key := filepath.Base(header.Filename)
writer, err := bucket.NewWriter(ctx, key, nil)
if err != nil {
http.Error(w, "Failed to create blob writer: "+err.Error(), http.StatusInternalServerError)
return
}
written, err := io.Copy(writer, file)
if err != nil {
writer.Close() // ignore error here, main err is from copy
http.Error(w, "Failed to write to blob: "+err.Error(), http.StatusInternalServerError)
return
}
err = writer.Close()
if err != nil {
http.Error(w, "Failed to close blob writer: "+err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
fmt.Fprintf(w, "Uploaded %s (%d bytes) to blob successfully\n", key, written)
}
}
func main() {
// Open local directory as blob bucket
ctx := context.Background()
bucket, err := blob.OpenBucket(ctx, "file:///tmp/mybucket")
if err != nil {
panic(fmt.Errorf("failed to open fileblob bucket: %w", err))
}
defer bucket.Close()
http.HandleFunc("/upload", uploadHandler(bucket))
fmt.Println("Listening on :8080...")
err = http.ListenAndServe(":8080", nil)
if err != nil {
panic(err)
}
}
On my machine (MacBook Pro M1 Max) it takes about ~28 sec e2e, roughly a 3.5x slower.
The same slowdown is noted with mem and s3.
Expected behavior
A negligible penalty from the abstraction to a 1.5x max slowdown.
Version
Additional context
Describe the bug
The upload performance when using the drivers (mem, file, s3 the only one tested) is very slow compared to direct use of native/alternative package.
To Reproduce
We will upload a large file ~4.5GB via
curlcurl -X POST -F "file=@large.bin" http://localhost:8080/uploadConsider this file upload to local file:
On my machine (MacBook Pro M1 Max) it takes about ~8 sec e2e.
The equivalent using
blobOn my machine (MacBook Pro M1 Max) it takes about ~28 sec e2e, roughly a 3.5x slower.
The same slowdown is noted with
memands3.Expected behavior
A negligible penalty from the abstraction to a 1.5x max slowdown.
Version
Additional context