git.stevedylan.dev

apps/blobs/s3client.go 5.5 K raw

package main

import (
	"context"
	"fmt"
	"io"
	"net/url"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/credentials"
	"github.com/aws/aws-sdk-go-v2/service/s3"
	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
)

type S3Client struct {
	s3         *s3.Client
	presigner  *s3.PresignClient
	publicURLs map[string]string
	presignTTL time.Duration
}

type BucketInfo struct {
	Name         string
	CreationDate string
}

type ObjectInfo struct {
	Key          string
	Name         string
	Size         int64
	LastModified string
	ETag         string
}

type ObjectMeta struct {
	Key           string
	ContentType   string
	Size          int64
	LastModified  string
	ETag          string
}

func NewS3Client(endpoint, region, accessKey, secretKey string, publicURLs map[string]string, presignTTL time.Duration) (*S3Client, error) {
	if strings.TrimSpace(accessKey) == "" || strings.TrimSpace(secretKey) == "" {
		return nil, fmt.Errorf("S3 access key and secret key are required")
	}
	if strings.TrimSpace(endpoint) == "" {
		return nil, fmt.Errorf("S3 endpoint is required (set S3_ENDPOINT or R2_ACCOUNT_ID)")
	}
	if region == "" {
		region = "auto"
	}
	cfg := aws.Config{
		Region:      region,
		Credentials: credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""),
	}
	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
		o.BaseEndpoint = aws.String(endpoint)
		o.UsePathStyle = true
	})
	if presignTTL <= 0 {
		presignTTL = time.Hour
	}
	return &S3Client{
		s3:         client,
		presigner:  s3.NewPresignClient(client),
		publicURLs: publicURLs,
		presignTTL: presignTTL,
	}, nil
}

func (c *S3Client) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
	out, err := c.s3.ListBuckets(ctx, &s3.ListBucketsInput{})
	if err != nil {
		return nil, err
	}
	buckets := make([]BucketInfo, 0, len(out.Buckets))
	for _, b := range out.Buckets {
		bi := BucketInfo{Name: aws.ToString(b.Name)}
		if b.CreationDate != nil {
			bi.CreationDate = b.CreationDate.UTC().Format("2006-01-02 15:04:05")
		}
		buckets = append(buckets, bi)
	}
	return buckets, nil
}

func (c *S3Client) List(ctx context.Context, bucket, prefix string) ([]string, []ObjectInfo, error) {
	folders := []string{}
	files := []ObjectInfo{}
	var token *string
	for {
		out, err := c.s3.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
			Bucket:            aws.String(bucket),
			Prefix:            aws.String(prefix),
			Delimiter:         aws.String("/"),
			ContinuationToken: token,
		})
		if err != nil {
			return nil, nil, err
		}
		for _, cp := range out.CommonPrefixes {
			folders = append(folders, aws.ToString(cp.Prefix))
		}
		for _, obj := range out.Contents {
			key := aws.ToString(obj.Key)
			if key == prefix {
				continue
			}
			name := strings.TrimPrefix(key, prefix)
			oi := ObjectInfo{
				Key:  key,
				Name: name,
				ETag: strings.Trim(aws.ToString(obj.ETag), `"`),
			}
			if obj.Size != nil {
				oi.Size = *obj.Size
			}
			if obj.LastModified != nil {
				oi.LastModified = obj.LastModified.UTC().Format("2006-01-02 15:04:05")
			}
			files = append(files, oi)
		}
		if out.IsTruncated == nil || !*out.IsTruncated {
			break
		}
		token = out.NextContinuationToken
	}
	return folders, files, nil
}

func (c *S3Client) Head(ctx context.Context, bucket, key string) (*ObjectMeta, error) {
	out, err := c.s3.HeadObject(ctx, &s3.HeadObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
	})
	if err != nil {
		return nil, err
	}
	m := &ObjectMeta{
		Key:         key,
		ContentType: aws.ToString(out.ContentType),
		ETag:        strings.Trim(aws.ToString(out.ETag), `"`),
	}
	if out.ContentLength != nil {
		m.Size = *out.ContentLength
	}
	if out.LastModified != nil {
		m.LastModified = out.LastModified.UTC().Format("2006-01-02 15:04:05")
	}
	return m, nil
}

func (c *S3Client) Get(ctx context.Context, bucket, key string) (io.ReadCloser, *ObjectMeta, error) {
	out, err := c.s3.GetObject(ctx, &s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
	})
	if err != nil {
		return nil, nil, err
	}
	m := &ObjectMeta{
		Key:         key,
		ContentType: aws.ToString(out.ContentType),
		ETag:        strings.Trim(aws.ToString(out.ETag), `"`),
	}
	if out.ContentLength != nil {
		m.Size = *out.ContentLength
	}
	if out.LastModified != nil {
		m.LastModified = out.LastModified.UTC().Format("2006-01-02 15:04:05")
	}
	return out.Body, m, nil
}

func (c *S3Client) Put(ctx context.Context, bucket, key, contentType string, body io.Reader, size int64) error {
	in := &s3.PutObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
		Body:   body,
	}
	if contentType != "" {
		in.ContentType = aws.String(contentType)
	}
	if size > 0 {
		in.ContentLength = aws.Int64(size)
	}
	_, err := c.s3.PutObject(ctx, in)
	return err
}

func (c *S3Client) Delete(ctx context.Context, bucket, key string) error {
	_, err := c.s3.DeleteObject(ctx, &s3.DeleteObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
	})
	return err
}

func (c *S3Client) PresignGet(ctx context.Context, bucket, key string) (string, error) {
	out, err := c.presigner.PresignGetObject(ctx, &s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
	}, s3.WithPresignExpires(c.presignTTL))
	if err != nil {
		return "", err
	}
	return out.URL, nil
}

func (c *S3Client) PublicURL(bucket, key string) (string, bool) {
	prefix, ok := c.publicURLs[bucket]
	if !ok || prefix == "" {
		return "", false
	}
	return strings.TrimRight(prefix, "/") + "/" + url.PathEscape(key), true
}

var _ s3types.Object

1	package main
2
3	import (
4	"context"
5	"fmt"
6	"io"
7	"net/url"
8	"strings"
9	"time"
10
11	"github.com/aws/aws-sdk-go-v2/aws"
12	"github.com/aws/aws-sdk-go-v2/credentials"
13	"github.com/aws/aws-sdk-go-v2/service/s3"
14	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
15	)
16
17	type S3Client struct {
18	s3 *s3.Client
19	presigner *s3.PresignClient
20	publicURLs map[string]string
21	presignTTL time.Duration
22	}
23
24	type BucketInfo struct {
25	Name string
26	CreationDate string
27	}
28
29	type ObjectInfo struct {
30	Key string
31	Name string
32	Size int64
33	LastModified string
34	ETag string
35	}
36
37	type ObjectMeta struct {
38	Key string
39	ContentType string
40	Size int64
41	LastModified string
42	ETag string
43	}
44
45	func NewS3Client(endpoint, region, accessKey, secretKey string, publicURLs map[string]string, presignTTL time.Duration) (*S3Client, error) {
46	if strings.TrimSpace(accessKey) == "" \|\| strings.TrimSpace(secretKey) == "" {
47	return nil, fmt.Errorf("S3 access key and secret key are required")
48	}
49	if strings.TrimSpace(endpoint) == "" {
50	return nil, fmt.Errorf("S3 endpoint is required (set S3_ENDPOINT or R2_ACCOUNT_ID)")
51	}
52	if region == "" {
53	region = "auto"
54	}
55	cfg := aws.Config{
56	Region: region,
57	Credentials: credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""),
58	}
59	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
60	o.BaseEndpoint = aws.String(endpoint)
61	o.UsePathStyle = true
62	})
63	if presignTTL <= 0 {
64	presignTTL = time.Hour
65	}
66	return &S3Client{
67	s3: client,
68	presigner: s3.NewPresignClient(client),
69	publicURLs: publicURLs,
70	presignTTL: presignTTL,
71	}, nil
72	}
73
74	func (c *S3Client) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
75	out, err := c.s3.ListBuckets(ctx, &s3.ListBucketsInput{})
76	if err != nil {
77	return nil, err
78	}
79	buckets := make([]BucketInfo, 0, len(out.Buckets))
80	for _, b := range out.Buckets {
81	bi := BucketInfo{Name: aws.ToString(b.Name)}
82	if b.CreationDate != nil {
83	bi.CreationDate = b.CreationDate.UTC().Format("2006-01-02 15:04:05")
84	}
85	buckets = append(buckets, bi)
86	}
87	return buckets, nil
88	}
89
90	func (c *S3Client) List(ctx context.Context, bucket, prefix string) ([]string, []ObjectInfo, error) {
91	folders := []string{}
92	files := []ObjectInfo{}
93	var token *string
94	for {
95	out, err := c.s3.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
96	Bucket: aws.String(bucket),
97	Prefix: aws.String(prefix),
98	Delimiter: aws.String("/"),
99	ContinuationToken: token,
100	})
101	if err != nil {
102	return nil, nil, err
103	}
104	for _, cp := range out.CommonPrefixes {
105	folders = append(folders, aws.ToString(cp.Prefix))
106	}
107	for _, obj := range out.Contents {
108	key := aws.ToString(obj.Key)
109	if key == prefix {
110	continue
111	}
112	name := strings.TrimPrefix(key, prefix)
113	oi := ObjectInfo{
114	Key: key,
115	Name: name,
116	ETag: strings.Trim(aws.ToString(obj.ETag), `"`),
117	}
118	if obj.Size != nil {
119	oi.Size = *obj.Size
120	}
121	if obj.LastModified != nil {
122	oi.LastModified = obj.LastModified.UTC().Format("2006-01-02 15:04:05")
123	}
124	files = append(files, oi)
125	}
126	if out.IsTruncated == nil \|\| !*out.IsTruncated {
127	break
128	}
129	token = out.NextContinuationToken
130	}
131	return folders, files, nil
132	}
133
134	func (c S3Client) Head(ctx context.Context, bucket, key string) (ObjectMeta, error) {
135	out, err := c.s3.HeadObject(ctx, &s3.HeadObjectInput{
136	Bucket: aws.String(bucket),
137	Key: aws.String(key),
138	})
139	if err != nil {
140	return nil, err
141	}
142	m := &ObjectMeta{
143	Key: key,
144	ContentType: aws.ToString(out.ContentType),
145	ETag: strings.Trim(aws.ToString(out.ETag), `"`),
146	}
147	if out.ContentLength != nil {
148	m.Size = *out.ContentLength
149	}
150	if out.LastModified != nil {
151	m.LastModified = out.LastModified.UTC().Format("2006-01-02 15:04:05")
152	}
153	return m, nil
154	}
155
156	func (c S3Client) Get(ctx context.Context, bucket, key string) (io.ReadCloser, ObjectMeta, error) {
157	out, err := c.s3.GetObject(ctx, &s3.GetObjectInput{
158	Bucket: aws.String(bucket),
159	Key: aws.String(key),
160	})
161	if err != nil {
162	return nil, nil, err
163	}
164	m := &ObjectMeta{
165	Key: key,
166	ContentType: aws.ToString(out.ContentType),
167	ETag: strings.Trim(aws.ToString(out.ETag), `"`),
168	}
169	if out.ContentLength != nil {
170	m.Size = *out.ContentLength
171	}
172	if out.LastModified != nil {
173	m.LastModified = out.LastModified.UTC().Format("2006-01-02 15:04:05")
174	}
175	return out.Body, m, nil
176	}
177
178	func (c *S3Client) Put(ctx context.Context, bucket, key, contentType string, body io.Reader, size int64) error {
179	in := &s3.PutObjectInput{
180	Bucket: aws.String(bucket),
181	Key: aws.String(key),
182	Body: body,
183	}
184	if contentType != "" {
185	in.ContentType = aws.String(contentType)
186	}
187	if size > 0 {
188	in.ContentLength = aws.Int64(size)
189	}
190	_, err := c.s3.PutObject(ctx, in)
191	return err
192	}
193
194	func (c *S3Client) Delete(ctx context.Context, bucket, key string) error {
195	_, err := c.s3.DeleteObject(ctx, &s3.DeleteObjectInput{
196	Bucket: aws.String(bucket),
197	Key: aws.String(key),
198	})
199	return err
200	}
201
202	func (c *S3Client) PresignGet(ctx context.Context, bucket, key string) (string, error) {
203	out, err := c.presigner.PresignGetObject(ctx, &s3.GetObjectInput{
204	Bucket: aws.String(bucket),
205	Key: aws.String(key),
206	}, s3.WithPresignExpires(c.presignTTL))
207	if err != nil {
208	return "", err
209	}
210	return out.URL, nil
211	}
212
213	func (c *S3Client) PublicURL(bucket, key string) (string, bool) {
214	prefix, ok := c.publicURLs[bucket]
215	if !ok \|\| prefix == "" {
216	return "", false
217	}
218	return strings.TrimRight(prefix, "/") + "/" + url.PathEscape(key), true
219	}
220
221	var _ s3types.Object