* Added push mirror model. * Integrated push mirror into queue. * Moved methods into own file. * Added basic implementation. * Mirror wiki too. * Removed duplicated method. * Get url for different remotes. * Added migration. * Unified remote url access. * Add/Remove push mirror remotes. * Prevent hangs with missing credentials. * Moved code between files. * Changed sanitizer interface. * Added push mirror backend methods. * Only update the mirror remote. * Limit refs on push. * Added UI part. * Added missing table. * Delete mirror if repository gets removed. * Changed signature. Handle object errors. * Added upload method. * Added "upload" unit tests. * Added transfer adapter unit tests. * Send correct headers. * Added pushing of LFS objects. * Added more logging. * Simpler body handling. * Process files in batches to reduce HTTP calls. * Added created timestamp. * Fixed invalid column name. * Changed name to prevent xorm auto setting. * Remove table header im empty. * Strip exit code from error message. * Added docs page about mirroring. * Fixed date. * Fixed merge errors. * Moved test to integrations. * Added push mirror test. * Added test.
		
			
				
	
	
		
			137 lines
		
	
	
		
			3.3 KiB
		
	
	
	
		
			Go
		
	
	
	
	
	
			
		
		
	
	
			137 lines
		
	
	
		
			3.3 KiB
		
	
	
	
		
			Go
		
	
	
	
	
	
| // Copyright 2021 The Gitea Authors. All rights reserved.
 | |
| // Use of this source code is governed by a MIT-style
 | |
| // license that can be found in the LICENSE file.
 | |
| 
 | |
| package lfs
 | |
| 
 | |
| import (
 | |
| 	"bytes"
 | |
| 	"context"
 | |
| 	"errors"
 | |
| 	"fmt"
 | |
| 	"io"
 | |
| 	"net/http"
 | |
| 
 | |
| 	"code.gitea.io/gitea/modules/log"
 | |
| 
 | |
| 	jsoniter "github.com/json-iterator/go"
 | |
| )
 | |
| 
 | |
| // TransferAdapter represents an adapter for downloading/uploading LFS objects
 | |
| type TransferAdapter interface {
 | |
| 	Name() string
 | |
| 	Download(ctx context.Context, l *Link) (io.ReadCloser, error)
 | |
| 	Upload(ctx context.Context, l *Link, p Pointer, r io.Reader) error
 | |
| 	Verify(ctx context.Context, l *Link, p Pointer) error
 | |
| }
 | |
| 
 | |
| // BasicTransferAdapter implements the "basic" adapter
 | |
| type BasicTransferAdapter struct {
 | |
| 	client *http.Client
 | |
| }
 | |
| 
 | |
| // Name returns the name of the adapter
 | |
| func (a *BasicTransferAdapter) Name() string {
 | |
| 	return "basic"
 | |
| }
 | |
| 
 | |
| // Download reads the download location and downloads the data
 | |
| func (a *BasicTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCloser, error) {
 | |
| 	resp, err := a.performRequest(ctx, "GET", l, nil, nil)
 | |
| 	if err != nil {
 | |
| 		return nil, err
 | |
| 	}
 | |
| 	return resp.Body, nil
 | |
| }
 | |
| 
 | |
| // Upload sends the content to the LFS server
 | |
| func (a *BasicTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r io.Reader) error {
 | |
| 	_, err := a.performRequest(ctx, "PUT", l, r, func(req *http.Request) {
 | |
| 		if len(req.Header.Get("Content-Type")) == 0 {
 | |
| 			req.Header.Set("Content-Type", "application/octet-stream")
 | |
| 		}
 | |
| 
 | |
| 		if req.Header.Get("Transfer-Encoding") == "chunked" {
 | |
| 			req.TransferEncoding = []string{"chunked"}
 | |
| 		}
 | |
| 
 | |
| 		req.ContentLength = p.Size
 | |
| 	})
 | |
| 	if err != nil {
 | |
| 		return err
 | |
| 	}
 | |
| 	return nil
 | |
| }
 | |
| 
 | |
| // Verify calls the verify handler on the LFS server
 | |
| func (a *BasicTransferAdapter) Verify(ctx context.Context, l *Link, p Pointer) error {
 | |
| 	b, err := jsoniter.Marshal(p)
 | |
| 	if err != nil {
 | |
| 		log.Error("Error encoding json: %v", err)
 | |
| 		return err
 | |
| 	}
 | |
| 
 | |
| 	_, err = a.performRequest(ctx, "POST", l, bytes.NewReader(b), func(req *http.Request) {
 | |
| 		req.Header.Set("Content-Type", MediaType)
 | |
| 	})
 | |
| 	if err != nil {
 | |
| 		return err
 | |
| 	}
 | |
| 	return nil
 | |
| }
 | |
| 
 | |
| func (a *BasicTransferAdapter) performRequest(ctx context.Context, method string, l *Link, body io.Reader, callback func(*http.Request)) (*http.Response, error) {
 | |
| 	log.Trace("Calling: %s %s", method, l.Href)
 | |
| 
 | |
| 	req, err := http.NewRequestWithContext(ctx, method, l.Href, body)
 | |
| 	if err != nil {
 | |
| 		log.Error("Error creating request: %v", err)
 | |
| 		return nil, err
 | |
| 	}
 | |
| 	for key, value := range l.Header {
 | |
| 		req.Header.Set(key, value)
 | |
| 	}
 | |
| 	req.Header.Set("Accept", MediaType)
 | |
| 
 | |
| 	if callback != nil {
 | |
| 		callback(req)
 | |
| 	}
 | |
| 
 | |
| 	res, err := a.client.Do(req)
 | |
| 	if err != nil {
 | |
| 		select {
 | |
| 		case <-ctx.Done():
 | |
| 			return res, ctx.Err()
 | |
| 		default:
 | |
| 		}
 | |
| 		log.Error("Error while processing request: %v", err)
 | |
| 		return res, err
 | |
| 	}
 | |
| 
 | |
| 	if res.StatusCode != http.StatusOK {
 | |
| 		return res, handleErrorResponse(res)
 | |
| 	}
 | |
| 
 | |
| 	return res, nil
 | |
| }
 | |
| 
 | |
| func handleErrorResponse(resp *http.Response) error {
 | |
| 	defer resp.Body.Close()
 | |
| 
 | |
| 	er, err := decodeReponseError(resp.Body)
 | |
| 	if err != nil {
 | |
| 		return fmt.Errorf("Request failed with status %s", resp.Status)
 | |
| 	}
 | |
| 	log.Trace("ErrorRespone: %v", er)
 | |
| 	return errors.New(er.Message)
 | |
| }
 | |
| 
 | |
| func decodeReponseError(r io.Reader) (ErrorResponse, error) {
 | |
| 	var er ErrorResponse
 | |
| 	err := jsoniter.NewDecoder(r).Decode(&er)
 | |
| 	if err != nil {
 | |
| 		log.Error("Error decoding json: %v", err)
 | |
| 	}
 | |
| 	return er, err
 | |
| }
 |