Optimize memory usage when stream flush and save (#659)

* use io.Copy from stream temp file to zip Writer

* fix nil

* log

* build

* delete log

* fix compatibility for office

* Update go module

Co-authored-by: lijingfeng <lijingfeng@laiye.com>
Co-authored-by: xuri <xuri.me@gmail.com>

excelize.go

@@ -36,6 +36,7 @@ type File struct {
 	xmlAttr          map[string][]xml.Attr
 	checked          map[string]bool
 	sheetMap         map[string]string
+	streams          map[string]*StreamWriter
 	CalcChain        *xlsxCalcChain
 	Comments         map[string]*xlsxComments
 	ContentTypes     *xlsxTypes

file.go

@@ -110,6 +110,26 @@ func (f *File) WriteToBuffer() (*bytes.Buffer, error) {
 	f.sharedStringsWriter()
 	f.styleSheetWriter()
 
+	for path, stream := range f.streams {
+		fi, err := zw.Create(path)
+		if err != nil {
+			zw.Close()
+			return buf, err
+		}
+		var from io.Reader
+		from, err = stream.rawData.Reader()
+		if err != nil {
+			stream.rawData.Close()
+			return buf, err
+		}
+		_, err = io.Copy(fi, from)
+		if err != nil {
+			zw.Close()
+			return buf, err
+		}
+		stream.rawData.Close()
+	}
+
 	for path, content := range f.XLSX {
 		fi, err := zw.Create(path)
 		if err != nil {

go.mod

@@ -1,6 +1,6 @@
 module github.com/360EntSecGroup-Skylar/excelize/v2
 
-go 1.15
+go 1.11
 
 require (
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826

stream.go

@@ -85,6 +85,13 @@ func (f *File) NewStreamWriter(sheet string) (*StreamWriter, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	sheetXML := fmt.Sprintf("xl/worksheets/sheet%d.xml", sw.SheetID)
+	if f.streams == nil {
+		f.streams = make(map[string]*StreamWriter)
+	}
+	f.streams[sheetXML] = sw
+
 	sw.rawData.WriteString(XMLHeader + `<worksheet` + templateNamespaceIDMap)
 	bulkAppendFields(&sw.rawData, sw.worksheet, 2, 6)
 	sw.rawData.WriteString(`<sheetData>`)
@@ -387,13 +394,8 @@ func (sw *StreamWriter) Flush() error {
 	sheetXML := fmt.Sprintf("xl/worksheets/sheet%d.xml", sw.SheetID)
 	delete(sw.File.Sheet, sheetXML)
 	delete(sw.File.checked, sheetXML)
+	delete(sw.File.XLSX, sheetXML)
 
-	defer sw.rawData.Close()
-	b, err := sw.rawData.Bytes()
-	if err != nil {
-		return err
-	}
-	sw.File.XLSX[sheetXML] = b
 	return nil
 }
 
@@ -444,36 +446,6 @@ func (bw *bufferedWriter) Reader() (io.Reader, error) {
 	return io.NewSectionReader(bw.tmp, 0, fi.Size()), nil
 }
 
-// Bytes returns the entire content of the bufferedWriter. If a temp file is
-// used, Bytes will efficiently allocate a buffer to prevent re-allocations.
-func (bw *bufferedWriter) Bytes() ([]byte, error) {
-	if bw.tmp == nil {
-		return bw.buf.Bytes(), nil
-	}
-
-	if err := bw.Flush(); err != nil {
-		return nil, err
-	}
-
-	var buf bytes.Buffer
-	if fi, err := bw.tmp.Stat(); err == nil {
-		if size := fi.Size() + bytes.MinRead; size > bytes.MinRead {
-			if int64(int(size)) == size {
-				buf.Grow(int(size))
-			} else {
-				return nil, bytes.ErrTooLarge
-			}
-		}
-	}
-
-	if _, err := bw.tmp.Seek(0, 0); err != nil {
-		return nil, err
-	}
-
-	_, err := buf.ReadFrom(bw.tmp)
-	return buf.Bytes(), err
-}
-
 // Sync will write the in-memory buffer to a temp file, if the in-memory
 // buffer has grown large enough. Any error will be returned.
 func (bw *bufferedWriter) Sync() (err error) {