From f7125ab61aaa02fd4c7ab0062a2dc9a57726e2ec Mon Sep 17 00:00:00 2001
From: Henrique Pimentel <66185935+HenriquerPimentel@users.noreply.github.com>
Date: Thu, 6 Jun 2024 09:06:59 +0100
Subject: [PATCH] Add `MAX_ROWS` option for CSV rendering (#30268)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This solution implements a new config variable MAX_ROWS, which
corresponds to the “Maximum allowed rows to render CSV files. (0 for no
limit)” and rewrites the Render function for CSV files in markup module.
Now the render function only reads the file once, having MAX_FILE_SIZE+1
as a reader limit and MAX_ROWS as a row limit. When the file is larger
than MAX_FILE_SIZE or has more rows than MAX_ROWS, it only renders until
the limit, and displays a user-friendly warning informing that the
rendered data is not complete, in the user's language.

---

Previously, when a CSV file was larger than the limit, the render
function lost its function to render the code. There were also multiple
reads to the file, in order to determine its size and render or
pre-render.

The warning: ![image](https://s3.amazonaws.com/i.snag.gy/vcKh90.jpg)
---
 custom/conf/app.example.ini    |  3 ++
 modules/markup/csv/csv.go      | 94 +++++++++++++---------------------
 modules/markup/csv/csv_test.go | 10 ----
 modules/setting/ui.go          |  3 ++
 4 files changed, 41 insertions(+), 69 deletions(-)

diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini
index 7677168d8..e619aae72 100644
--- a/custom/conf/app.example.ini
+++ b/custom/conf/app.example.ini
@@ -1334,6 +1334,9 @@ LEVEL = Info
 ;;
 ;; Maximum allowed file size in bytes to render CSV files as table. (Set to 0 for no limit).
 ;MAX_FILE_SIZE = 524288
+;;
+;; Maximum allowed rows to render CSV files. (Set to 0 for no limit)
+;MAX_ROWS = 2500
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
diff --git a/modules/markup/csv/csv.go b/modules/markup/csv/csv.go
index 1dd26eb8a..3d952b0de 100644
--- a/modules/markup/csv/csv.go
+++ b/modules/markup/csv/csv.go
@@ -5,8 +5,6 @@ package markup
 
 import (
 	"bufio"
-	"bytes"
-	"fmt"
 	"html"
 	"io"
 	"regexp"
@@ -15,6 +13,8 @@ import (
 	"code.gitea.io/gitea/modules/csv"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/translation"
+	"code.gitea.io/gitea/modules/util"
 )
 
 func init() {
@@ -81,86 +81,38 @@ func writeField(w io.Writer, element, class, field string) error {
 func (r Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
 	tmpBlock := bufio.NewWriter(output)
 	maxSize := setting.UI.CSV.MaxFileSize
+	maxRows := setting.UI.CSV.MaxRows
 
-	if maxSize == 0 {
-		return r.tableRender(ctx, input, tmpBlock)
+	if maxSize != 0 {
+		input = io.LimitReader(input, maxSize+1)
 	}
 
-	rawBytes, err := io.ReadAll(io.LimitReader(input, maxSize+1))
-	if err != nil {
-		return err
-	}
-
-	if int64(len(rawBytes)) <= maxSize {
-		return r.tableRender(ctx, bytes.NewReader(rawBytes), tmpBlock)
-	}
-	return r.fallbackRender(io.MultiReader(bytes.NewReader(rawBytes), input), tmpBlock)
-}
-
-func (Renderer) fallbackRender(input io.Reader, tmpBlock *bufio.Writer) error {
-	_, err := tmpBlock.WriteString("<pre>")
-	if err != nil {
-		return err
-	}
-
-	scan := bufio.NewScanner(input)
-	scan.Split(bufio.ScanRunes)
-	for scan.Scan() {
-		switch scan.Text() {
-		case `&`:
-			_, err = tmpBlock.WriteString("&amp;")
-		case `'`:
-			_, err = tmpBlock.WriteString("&#39;") // "&#39;" is shorter than "&apos;" and apos was not in HTML until HTML5.
-		case `<`:
-			_, err = tmpBlock.WriteString("&lt;")
-		case `>`:
-			_, err = tmpBlock.WriteString("&gt;")
-		case `"`:
-			_, err = tmpBlock.WriteString("&#34;") // "&#34;" is shorter than "&quot;".
-		default:
-			_, err = tmpBlock.Write(scan.Bytes())
-		}
-		if err != nil {
-			return err
-		}
-	}
-	if err = scan.Err(); err != nil {
-		return fmt.Errorf("fallbackRender scan: %w", err)
-	}
-
-	_, err = tmpBlock.WriteString("</pre>")
-	if err != nil {
-		return err
-	}
-	return tmpBlock.Flush()
-}
-
-func (Renderer) tableRender(ctx *markup.RenderContext, input io.Reader, tmpBlock *bufio.Writer) error {
 	rd, err := csv.CreateReaderAndDetermineDelimiter(ctx, input)
 	if err != nil {
 		return err
 	}
-
 	if _, err := tmpBlock.WriteString(`<table class="data-table">`); err != nil {
 		return err
 	}
-	row := 1
+
+	row := 0
 	for {
 		fields, err := rd.Read()
-		if err == io.EOF {
+		if err == io.EOF || (row >= maxRows && maxRows != 0) {
 			break
 		}
 		if err != nil {
 			continue
 		}
+
 		if _, err := tmpBlock.WriteString("<tr>"); err != nil {
 			return err
 		}
 		element := "td"
-		if row == 1 {
+		if row == 0 {
 			element = "th"
 		}
-		if err := writeField(tmpBlock, element, "line-num", strconv.Itoa(row)); err != nil {
+		if err := writeField(tmpBlock, element, "line-num", strconv.Itoa(row+1)); err != nil {
 			return err
 		}
 		for _, field := range fields {
@@ -174,8 +126,32 @@ func (Renderer) tableRender(ctx *markup.RenderContext, input io.Reader, tmpBlock
 
 		row++
 	}
+
 	if _, err = tmpBlock.WriteString("</table>"); err != nil {
 		return err
 	}
+
+	// Check if maxRows or maxSize is reached, and if true, warn.
+	if (row >= maxRows && maxRows != 0) || (rd.InputOffset() >= maxSize && maxSize != 0) {
+		warn := `<table class="data-table"><tr><td>`
+		rawLink := ` <a href="` + ctx.Links.RawLink() + `/` + util.PathEscapeSegments(ctx.RelativePath) + `">`
+
+		// Try to get the user translation
+		if locale, ok := ctx.Ctx.Value(translation.ContextKey).(translation.Locale); ok {
+			warn += locale.TrString("repo.file_too_large")
+			rawLink += locale.TrString("repo.file_view_raw")
+		} else {
+			warn += "The file is too large to be shown."
+			rawLink += "View Raw"
+		}
+
+		warn += rawLink + `</a></td></tr></table>`
+
+		// Write the HTML string to the output
+		if _, err := tmpBlock.WriteString(warn); err != nil {
+			return err
+		}
+	}
+
 	return tmpBlock.Flush()
 }
diff --git a/modules/markup/csv/csv_test.go b/modules/markup/csv/csv_test.go
index 3d12be477..8c07184b2 100644
--- a/modules/markup/csv/csv_test.go
+++ b/modules/markup/csv/csv_test.go
@@ -4,8 +4,6 @@
 package markup
 
 import (
-	"bufio"
-	"bytes"
 	"strings"
 	"testing"
 
@@ -31,12 +29,4 @@ func TestRenderCSV(t *testing.T) {
 		assert.NoError(t, err)
 		assert.EqualValues(t, v, buf.String())
 	}
-
-	t.Run("fallbackRender", func(t *testing.T) {
-		var buf bytes.Buffer
-		err := render.fallbackRender(strings.NewReader("1,<a>\n2,<b>"), bufio.NewWriter(&buf))
-		assert.NoError(t, err)
-		want := "<pre>1,&lt;a&gt;\n2,&lt;b&gt;</pre>"
-		assert.Equal(t, want, buf.String())
-	})
 }
diff --git a/modules/setting/ui.go b/modules/setting/ui.go
index 93855bca0..a8dc11d09 100644
--- a/modules/setting/ui.go
+++ b/modules/setting/ui.go
@@ -52,6 +52,7 @@ var UI = struct {
 
 	CSV struct {
 		MaxFileSize int64
+		MaxRows     int
 	} `ini:"ui.csv"`
 
 	Admin struct {
@@ -107,8 +108,10 @@ var UI = struct {
 	},
 	CSV: struct {
 		MaxFileSize int64
+		MaxRows     int
 	}{
 		MaxFileSize: 524288,
+		MaxRows:     2500,
 	},
 	Admin: struct {
 		UserPagingNum   int