Implement Cargo HTTP index (#24452)

This implements the HTTP index
[RFC](https://rust-lang.github.io/rfcs/2789-sparse-index.html) for Cargo
registries.

Currently this is a preview feature and you need to use the nightly of
`cargo`:

`cargo +nightly -Z sparse-registry update`

See https://github.com/rust-lang/cargo/issues/9069 for more information.

---------

Co-authored-by: Giteabot <teabot@gitea.io>

routers/api/packages/api.go

@@ -119,6 +119,12 @@ func CommonRoutes(ctx gocontext.Context) *web.Route {
 					r.Get("/owners", cargo.ListOwners)
 				})
 			})
+			r.Get("/config.json", cargo.RepositoryConfig)
+			r.Get("/1/{package}", cargo.EnumeratePackageVersions)
+			r.Get("/2/{package}", cargo.EnumeratePackageVersions)
+			// Use dummy placeholders because these parts are not of interest
+			r.Get("/3/{_}/{package}", cargo.EnumeratePackageVersions)
+			r.Get("/{_}/{__}/{package}", cargo.EnumeratePackageVersions)
 		}, reqPackageAccess(perm.AccessModeRead))
 		r.Group("/chef", func() {
 			r.Group("/api/v1", func() {

routers/api/packages/cargo/cargo.go

@@ -4,6 +4,7 @@
 package cargo
 
 import (
+	"errors"
 	"fmt"
 	"net/http"
 	"strconv"
@@ -45,6 +46,35 @@ func apiError(ctx *context.Context, status int, obj interface{}) {
 	})
 }
 
+// https://rust-lang.github.io/rfcs/2789-sparse-index.html
+func RepositoryConfig(ctx *context.Context) {
+	ctx.JSON(http.StatusOK, cargo_service.BuildConfig(ctx.Package.Owner))
+}
+
+func EnumeratePackageVersions(ctx *context.Context) {
+	p, err := packages_model.GetPackageByName(ctx, ctx.Package.Owner.ID, packages_model.TypeCargo, ctx.Params("package"))
+	if err != nil {
+		if errors.Is(err, util.ErrNotExist) {
+			apiError(ctx, http.StatusNotFound, err)
+		} else {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	b, err := cargo_service.BuildPackageIndex(ctx, p)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	if b == nil {
+		apiError(ctx, http.StatusNotFound, nil)
+		return
+	}
+
+	ctx.PlainTextBytes(http.StatusOK, b.Bytes())
+}
+
 type SearchResult struct {
 	Crates []*SearchResultCrate `json:"crates"`
 	Meta   SearchResultMeta     `json:"meta"`

services/packages/cargo/index.go

@@ -137,21 +137,21 @@ type IndexVersionEntry struct {
 	Links        string                     `json:"links,omitempty"`
 }
 
-func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, p *packages_model.Package) error {
+func BuildPackageIndex(ctx context.Context, p *packages_model.Package) (*bytes.Buffer, error) {
 	pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
 		PackageID: p.ID,
 		Sort:      packages_model.SortVersionAsc,
 	})
 	if err != nil {
-		return fmt.Errorf("SearchVersions[%s]: %w", p.Name, err)
+		return nil, fmt.Errorf("SearchVersions[%s]: %w", p.Name, err)
 	}
 	if len(pvs) == 0 {
-		return nil
+		return nil, nil
 	}
 
 	pds, err := packages_model.GetPackageDescriptors(ctx, pvs)
 	if err != nil {
-		return fmt.Errorf("GetPackageDescriptors[%s]: %w", p.Name, err)
+		return nil, fmt.Errorf("GetPackageDescriptors[%s]: %w", p.Name, err)
 	}
 
 	var b bytes.Buffer
@@ -179,14 +179,26 @@ func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUplo
 			Links:        metadata.Links,
 		})
 		if err != nil {
-			return err
+			return nil, err
 		}
 
 		b.Write(entry)
 		b.WriteString("\n")
 	}
 
-	return writeObjectToIndex(t, BuildPackagePath(pds[0].Package.LowerName), &b)
+	return &b, nil
+}
+
+func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, p *packages_model.Package) error {
+	b, err := BuildPackageIndex(ctx, p)
+	if err != nil {
+		return err
+	}
+	if b == nil {
+		return nil
+	}
+
+	return writeObjectToIndex(t, BuildPackagePath(p.LowerName), b)
 }
 
 func getOrCreateIndexRepository(ctx context.Context, doer, owner *user_model.User) (*repo_model.Repository, error) {
@@ -212,6 +224,13 @@ type Config struct {
 	APIURL      string `json:"api"`
 }
 
+func BuildConfig(owner *user_model.User) *Config {
+	return &Config{
+		DownloadURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo/api/v1/crates",
+		APIURL:      setting.AppURL + "api/packages/" + owner.Name + "/cargo",
+	}
+}
+
 func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository, doer, owner *user_model.User) error {
 	return alterRepositoryContent(
 		ctx,
@@ -220,10 +239,7 @@ func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository,
 		"Initialize Cargo Config",
 		func(t *files_service.TemporaryUploadRepository) error {
 			var b bytes.Buffer
-			err := json.NewEncoder(&b).Encode(Config{
+			err := json.NewEncoder(&b).Encode(BuildConfig(owner))
-				DownloadURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo/api/v1/crates",
-				APIURL:      setting.AppURL + "api/packages/" + owner.Name + "/cargo",
-			})
 			if err != nil {
 				return err
 			}

tests/integration/api_packages_cargo_test.go

@@ -98,7 +98,7 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) {
 	url := fmt.Sprintf("%s/api/v1/crates", root)
 
 	t.Run("Index", func(t *testing.T) {
-		t.Run("Config", func(t *testing.T) {
+		t.Run("Git/Config", func(t *testing.T) {
 			defer tests.PrintCurrentTest(t)()
 
 			content := readGitContent(t, cargo_service.ConfigFileName)
@@ -110,6 +110,20 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) {
 			assert.Equal(t, url, config.DownloadURL)
 			assert.Equal(t, root, config.APIURL)
 		})
+
+		t.Run("HTTP/Config", func(t *testing.T) {
+			defer tests.PrintCurrentTest(t)()
+
+			req := NewRequest(t, "GET", root+"/"+cargo_service.ConfigFileName)
+			resp := MakeRequest(t, req, http.StatusOK)
+
+			var config cargo_service.Config
+			err := json.Unmarshal(resp.Body.Bytes(), &config)
+			assert.NoError(t, err)
+
+			assert.Equal(t, url, config.DownloadURL)
+			assert.Equal(t, root, config.APIURL)
+		})
 	})
 
 	t.Run("Upload", func(t *testing.T) {
@@ -192,40 +206,72 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) {
 			MakeRequest(t, req, http.StatusConflict)
 
 			t.Run("Index", func(t *testing.T) {
-				t.Run("Entry", func(t *testing.T) {
+				t.Run("Git", func(t *testing.T) {
-					defer tests.PrintCurrentTest(t)()
+					t.Run("Entry", func(t *testing.T) {
+						defer tests.PrintCurrentTest(t)()
 
-					content := readGitContent(t, cargo_service.BuildPackagePath(packageName))
+						content := readGitContent(t, cargo_service.BuildPackagePath(packageName))
 
-					var entry cargo_service.IndexVersionEntry
+						var entry cargo_service.IndexVersionEntry
-					err := json.Unmarshal([]byte(content), &entry)
+						err := json.Unmarshal([]byte(content), &entry)
-					assert.NoError(t, err)
+						assert.NoError(t, err)
 
-					assert.Equal(t, packageName, entry.Name)
+						assert.Equal(t, packageName, entry.Name)
-					assert.Equal(t, packageVersion, entry.Version)
+						assert.Equal(t, packageVersion, entry.Version)
-					assert.Equal(t, pb.HashSHA256, entry.FileChecksum)
+						assert.Equal(t, pb.HashSHA256, entry.FileChecksum)
-					assert.False(t, entry.Yanked)
+						assert.False(t, entry.Yanked)
-					assert.Len(t, entry.Dependencies, 1)
+						assert.Len(t, entry.Dependencies, 1)
-					dep := entry.Dependencies[0]
+						dep := entry.Dependencies[0]
-					assert.Equal(t, "dep", dep.Name)
+						assert.Equal(t, "dep", dep.Name)
-					assert.Equal(t, "1.0", dep.Req)
+						assert.Equal(t, "1.0", dep.Req)
-					assert.Equal(t, "normal", dep.Kind)
+						assert.Equal(t, "normal", dep.Kind)
-					assert.True(t, dep.DefaultFeatures)
+						assert.True(t, dep.DefaultFeatures)
-					assert.Empty(t, dep.Features)
+						assert.Empty(t, dep.Features)
-					assert.False(t, dep.Optional)
+						assert.False(t, dep.Optional)
-					assert.Nil(t, dep.Target)
+						assert.Nil(t, dep.Target)
-					assert.NotNil(t, dep.Registry)
+						assert.NotNil(t, dep.Registry)
-					assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry)
+						assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry)
-					assert.Nil(t, dep.Package)
+						assert.Nil(t, dep.Package)
+					})
+
+					t.Run("Rebuild", func(t *testing.T) {
+						defer tests.PrintCurrentTest(t)()
+
+						err := cargo_service.RebuildIndex(db.DefaultContext, user, user)
+						assert.NoError(t, err)
+
+						_ = readGitContent(t, cargo_service.BuildPackagePath(packageName))
+					})
 				})
 
-				t.Run("Rebuild", func(t *testing.T) {
+				t.Run("HTTP", func(t *testing.T) {
-					defer tests.PrintCurrentTest(t)()
+					t.Run("Entry", func(t *testing.T) {
+						defer tests.PrintCurrentTest(t)()
 
-					err := cargo_service.RebuildIndex(db.DefaultContext, user, user)
+						req := NewRequest(t, "GET", root+"/"+cargo_service.BuildPackagePath(packageName))
-					assert.NoError(t, err)
+						resp := MakeRequest(t, req, http.StatusOK)
 
-					_ = readGitContent(t, cargo_service.BuildPackagePath(packageName))
+						var entry cargo_service.IndexVersionEntry
+						err := json.Unmarshal(resp.Body.Bytes(), &entry)
+						assert.NoError(t, err)
+
+						assert.Equal(t, packageName, entry.Name)
+						assert.Equal(t, packageVersion, entry.Version)
+						assert.Equal(t, pb.HashSHA256, entry.FileChecksum)
+						assert.False(t, entry.Yanked)
+						assert.Len(t, entry.Dependencies, 1)
+						dep := entry.Dependencies[0]
+						assert.Equal(t, "dep", dep.Name)
+						assert.Equal(t, "1.0", dep.Req)
+						assert.Equal(t, "normal", dep.Kind)
+						assert.True(t, dep.DefaultFeatures)
+						assert.Empty(t, dep.Features)
+						assert.False(t, dep.Optional)
+						assert.Nil(t, dep.Target)
+						assert.NotNil(t, dep.Registry)
+						assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry)
+						assert.Nil(t, dep.Package)
+					})
 				})
 			})
 		})