Support nuspec manifest download for nuget packages (#28921)

Support downloading nuget nuspec manifest[^1]. This is useful for
renovate because it uses this api to find the corresponding repository

- Store nuspec along with nupkg on upload
- allow downloading nuspec
- add doctor command to add missing nuspec files 


[^1]:
https://learn.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-manifest-nuspec

---------

Co-authored-by: KN4CK3R <admin@oldschoolhack.me>

modules/packages/nuget/metadata.go

@@ -48,10 +48,11 @@ const maxNuspecFileSize = 3 * 1024 * 1024
 
 // Package represents a Nuget package
 type Package struct {
-	PackageType PackageType
+	PackageType   PackageType
-	ID          string
+	ID            string
-	Version     string
+	Version       string
-	Metadata    *Metadata
+	Metadata      *Metadata
+	NuspecContent *bytes.Buffer
 }
 
 // Metadata represents the metadata of a Nuget package
@@ -138,8 +139,9 @@ func ParsePackageMetaData(r io.ReaderAt, size int64) (*Package, error) {
 
 // ParseNuspecMetaData parses a Nuspec file to retrieve the metadata of a Nuget package
 func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) {
+	var nuspecBuf bytes.Buffer
 	var p nuspecPackage
-	if err := xml.NewDecoder(r).Decode(&p); err != nil {
+	if err := xml.NewDecoder(io.TeeReader(r, &nuspecBuf)).Decode(&p); err != nil {
 		return nil, err
 	}
 
@@ -212,10 +214,11 @@ func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) {
 		}
 	}
 	return &Package{
-		PackageType: packageType,
+		PackageType:   packageType,
-		ID:          p.Metadata.ID,
+		ID:            p.Metadata.ID,
-		Version:     toNormalizedVersion(v),
+		Version:       toNormalizedVersion(v),
-		Metadata:    m,
+		Metadata:      m,
+		NuspecContent: &nuspecBuf,
 	}, nil
 }
 

routers/api/packages/nuget/nuget.go

@@ -388,7 +388,8 @@ func EnumeratePackageVersionsV3(ctx *context.Context) {
 	ctx.JSON(http.StatusOK, resp)
 }
 
-// https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-content-nupkg
+// https://learn.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-manifest-nuspec
+// https://learn.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-content-nupkg
 func DownloadPackageFile(ctx *context.Context) {
 	packageName := ctx.Params("id")
 	packageVersion := ctx.Params("version")
@@ -431,7 +432,7 @@ func UploadPackage(ctx *context.Context) {
 		return
 	}
 
-	_, _, err := packages_service.CreatePackageAndAddFile(
+	pv, _, err := packages_service.CreatePackageAndAddFile(
 		ctx,
 		&packages_service.PackageCreationInfo{
 			PackageInfo: packages_service.PackageInfo{
@@ -465,6 +466,33 @@ func UploadPackage(ctx *context.Context) {
 		return
 	}
 
+	nuspecBuf, err := packages_module.CreateHashedBufferFromReaderWithSize(np.NuspecContent, np.NuspecContent.Len())
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	defer nuspecBuf.Close()
+
+	_, err = packages_service.AddFileToPackageVersionInternal(
+		ctx,
+		pv,
+		&packages_service.PackageFileCreationInfo{
+			PackageFileInfo: packages_service.PackageFileInfo{
+				Filename: strings.ToLower(fmt.Sprintf("%s.nuspec", np.ID)),
+			},
+			Data: nuspecBuf,
+		},
+	)
+	if err != nil {
+		switch err {
+		case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
+			apiError(ctx, http.StatusForbidden, err)
+		default:
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
 	ctx.Status(http.StatusCreated)
 }
 

tests/integration/api_packages_nuget_test.go

@@ -90,29 +90,33 @@ func TestPackageNuGet(t *testing.T) {
 	symbolFilename := "test.pdb"
 	symbolID := "d910bb6948bd4c6cb40155bcf52c3c94"
 
-	createPackage := func(id, version string) io.Reader {
+	createNuspec := func(id, version string) string {
+		return `<?xml version="1.0" encoding="utf-8"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
+	<metadata>
+		<id>` + id + `</id>
+		<version>` + version + `</version>
+		<authors>` + packageAuthors + `</authors>
+		<description>` + packageDescription + `</description>
+		<dependencies>
+			<group targetFramework=".NETStandard2.0">
+				<dependency id="Microsoft.CSharp" version="4.5.0" />
+			</group>
+		</dependencies>
+	</metadata>
+</package>`
+	}
+
+	createPackage := func(id, version string) *bytes.Buffer {
 		var buf bytes.Buffer
 		archive := zip.NewWriter(&buf)
 		w, _ := archive.Create("package.nuspec")
-		w.Write([]byte(`<?xml version="1.0" encoding="utf-8"?>
+		w.Write([]byte(createNuspec(id, version)))
-		<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
-			<metadata>
-				<id>` + id + `</id>
-				<version>` + version + `</version>
-				<authors>` + packageAuthors + `</authors>
-				<description>` + packageDescription + `</description>
-				<dependencies>
-					<group targetFramework=".NETStandard2.0">
-						<dependency id="Microsoft.CSharp" version="4.5.0" />
-					</group>
-				</dependencies>
-			</metadata>
-		</package>`))
 		archive.Close()
 		return &buf
 	}
 
-	content, _ := io.ReadAll(createPackage(packageName, packageVersion))
+	content := createPackage(packageName, packageVersion).Bytes()
 
 	url := fmt.Sprintf("/api/packages/%s/nuget", user.Name)
 
@@ -224,7 +228,7 @@ func TestPackageNuGet(t *testing.T) {
 
 			pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeNuGet)
 			assert.NoError(t, err)
-			assert.Len(t, pvs, 1)
+			assert.Len(t, pvs, 1, "Should have one version")
 
 			pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0])
 			assert.NoError(t, err)
@@ -235,13 +239,21 @@ func TestPackageNuGet(t *testing.T) {
 
 			pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID)
 			assert.NoError(t, err)
-			assert.Len(t, pfs, 1)
+			assert.Len(t, pfs, 2, "Should have 2 files: nuget and nuspec")
-			assert.Equal(t, fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion), pfs[0].Name)
+			for _, pf := range pfs {
-			assert.True(t, pfs[0].IsLead)
+				switch pf.Name {
+				case fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion):
+					assert.True(t, pf.IsLead)
 
-			pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID)
+					pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID)
-			assert.NoError(t, err)
+					assert.NoError(t, err)
-			assert.Equal(t, int64(len(content)), pb.Size)
+					assert.Equal(t, int64(len(content)), pb.Size)
+				case fmt.Sprintf("%s.nuspec", packageName):
+					assert.False(t, pf.IsLead)
+				default:
+					assert.Fail(t, "unexpected filename: %v", pf.Name)
+				}
+			}
 
 			req = NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)).
 				AddBasicAuth(user.Name)
@@ -302,16 +314,27 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`)
 
 			pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID)
 			assert.NoError(t, err)
-			assert.Len(t, pfs, 3)
+			assert.Len(t, pfs, 4, "Should have 4 files: nupkg, snupkg, nuspec and pdb")
 			for _, pf := range pfs {
 				switch pf.Name {
 				case fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion):
+					assert.True(t, pf.IsLead)
+
+					pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID)
+					assert.NoError(t, err)
+					assert.Equal(t, int64(412), pb.Size)
 				case fmt.Sprintf("%s.%s.snupkg", packageName, packageVersion):
 					assert.False(t, pf.IsLead)
 
 					pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID)
 					assert.NoError(t, err)
 					assert.Equal(t, int64(616), pb.Size)
+				case fmt.Sprintf("%s.nuspec", packageName):
+					assert.False(t, pf.IsLead)
+
+					pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID)
+					assert.NoError(t, err)
+					assert.Equal(t, int64(427), pb.Size)
 				case symbolFilename:
 					assert.False(t, pf.IsLead)
 
@@ -353,6 +376,12 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`)
 
 		assert.Equal(t, content, resp.Body.Bytes())
 
+		req = NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.nuspec", url, packageName, packageVersion, packageName)).
+			AddBasicAuth(user.Name)
+		resp = MakeRequest(t, req, http.StatusOK)
+
+		assert.Equal(t, createNuspec(packageName, packageVersion), resp.Body.String())
+
 		checkDownloadCount(1)
 
 		req = NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.%s.snupkg", url, packageName, packageVersion, packageName, packageVersion)).