Created
June 20, 2024 16:30
-
-
Save lzap/e5da341b89228749676f25a3651043e2 to your computer and use it in GitHub Desktop.
Memory-mapped artifact upload into Pulp in Go and oapi-gen Go HTTP client
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import ( | |
| "bytes" | |
| "context" | |
| "crypto/sha256" | |
| "encoding/hex" | |
| "fmt" | |
| "io" | |
| "mime/multipart" | |
| "os" | |
| "path/filepath" | |
| "golang.org/x/exp/mmap" | |
| ) | |
| // ArtifactsCreate creates an artifact from a file. It uses UNIX mmap to avoid buffering the file in memory. | |
| func (c *PulpService) ArtifactsCreate(ctx context.Context, filename string) (*ArtifactResponse, error) { | |
| at, err := mmap.Open(filename) | |
| if err != nil { | |
| return nil, err | |
| } | |
| file := MmapReader{at: at} | |
| defer file.Close() | |
| hash := sha256.New() | |
| if _, err := io.Copy(hash, &file); err != nil { | |
| return nil, err | |
| } | |
| sha256h := hash.Sum(nil) | |
| file.Reset() | |
| body := &bytes.Buffer{} | |
| writer := multipart.NewWriter(body) | |
| err = writer.WriteField("size", fmt.Sprintf("%d", at.Len())) | |
| if err != nil { | |
| return nil, err | |
| } | |
| err = writer.WriteField("sha256", hex.EncodeToString(sha256h[:])) | |
| if err != nil { | |
| return nil, err | |
| } | |
| part, err := writer.CreateFormFile("file", filepath.Base(filename)) | |
| if err != nil { | |
| return nil, err | |
| } | |
| _, err = io.Copy(part, &file) | |
| if err != nil { | |
| return nil, err | |
| } | |
| err = writer.Close() | |
| if err != nil { | |
| return nil, err | |
| } | |
| resp, err := c.cwr.ArtifactsCreateWithBodyWithResponse(ctx, c.dom, writer.FormDataContentType(), body, addAuthenticationHeader) | |
| if err != nil { | |
| return nil, err | |
| } | |
| if resp.JSON201 == nil { | |
| return nil, fmt.Errorf("unexpected response: %d, correlation id: %s, body: %s", resp.StatusCode(), resp.HTTPResponse.Header.Get("Correlation-Id"), string(resp.Body)) | |
| } | |
| return resp.JSON201, nil | |
| } | |
| type MmapReader struct { | |
| at *mmap.ReaderAt | |
| cur int | |
| } | |
| func (r *MmapReader) Read(p []byte) (n int, err error) { | |
| n, err = r.at.ReadAt(p, int64(r.cur)) | |
| r.cur += n | |
| return | |
| } | |
| func (r *MmapReader) Reset() { | |
| r.cur = 0 | |
| } | |
| func (r *MmapReader) Close() error { | |
| return r.at.Close() | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment