Skip to content

Instantly share code, notes, and snippets.

@olivoil

olivoil/file_uploader.dart

Last active August 13, 2024 03:46
Show Gist options
  • Save olivoil/36a69cf37f54d3a009540202cc413827 to your computer and use it in GitHub Desktop.
Save olivoil/36a69cf37f54d3a009540202cc413827 to your computer and use it in GitHub Desktop.
Download ZIP
Serverpod - Google Cloud Storage Interim
Raw
file_uploader.dart
import 'dart:convert';
import 'dart:typed_data';
import 'package:http/http.dart' as http;
/// The file uploader uploads files to Serverpod's cloud storage. On the server
/// you can setup a custom storage service, such as S3 or Google Cloud. To
/// directly upload a file, you first need to retrieve an upload description
/// from your server. After the file is uploaded, make sure to notify the server
/// by calling the verifyDirectFileUpload on the current Session object.
class GoogleCloudStorageUploader {
late final _UploadDescription _uploadDescription;
bool _attemptedUpload = false;
/// Creates a new FileUploader from an [uploadDescription] created by the
/// server.
GoogleCloudStorageUploader(String uploadDescription) {
_uploadDescription = _UploadDescription(uploadDescription);
}
/// Uploads a file contained by a [ByteData] object, returns true if
/// successful.
Future<bool> uploadByteData(ByteData byteData) async {
var stream = http.ByteStream.fromBytes(byteData.buffer.asUint8List());
return upload(stream, byteData.lengthInBytes);
}
/// Uploads a file from a [Stream], returns true if successful.
Future<bool> upload(Stream<List<int>> stream, int length) async {
if (_attemptedUpload) {
throw Exception(
'Data has already been uploaded using this FileUploader.');
}
_attemptedUpload = true;
if (_uploadDescription.type == _UploadType.binary) {
try {
var result = switch (_uploadDescription.httpMethod) {
'PUT' => await http.put(
_uploadDescription.url,
body: await _readStreamData(stream),
headers: _uploadDescription.headers,
),
_ => await http.post(
_uploadDescription.url,
body: await _readStreamData(stream),
headers: _uploadDescription.headers,
),
};
if (result.statusCode == 200) {
print('statusCode: ${result.statusCode}, body: ${result.body}');
}
return result.statusCode == 200;
} catch (e) {
return false;
}
} else if (_uploadDescription.type == _UploadType.multipart) {
// final stream = http.ByteStream(Stream.castFrom(file.openRead()));
// final length = await file.length();
// final stream = http.ByteStream.fromBytes(data.buffer.asUint8List());
// final length = await data.lengthInBytes;
var request = http.MultipartRequest('POST', _uploadDescription.url);
var multipartFile = http.MultipartFile(
_uploadDescription.field!, stream, length,
filename: _uploadDescription.fileName);
request.files.add(multipartFile);
for (var key in _uploadDescription.requestFields.keys) {
request.fields[key] = _uploadDescription.requestFields[key]!;
}
try {
var result = await request.send();
// var body = await _readBody(result.stream);
// print('body: $body');
return result.statusCode == 204;
} catch (e) {
return false;
}
}
throw UnimplementedError('Unknown upload type');
}
Future<List<int>> _readStreamData(Stream<List<int>> stream) async {
// TODO: Find more efficient solution?
var data = <int>[];
await for (var segment in stream) {
data += segment;
}
return data;
}
}
enum _UploadType {
binary,
multipart,
}
class _UploadDescription {
late _UploadType type;
late Uri url;
String? field;
String? fileName;
String? httpMethod = 'POST';
Map<String, String> headers = {};
Map<String, String> requestFields = {};
_UploadDescription(String description) {
var data = jsonDecode(description);
if (data['type'] == 'binary') {
type = _UploadType.binary;
} else if (data['type'] == 'multipart') {
type = _UploadType.multipart;
} else {
throw const FormatException('Missing type, can be binary or multipart');
}
httpMethod = data['httpMethod'];
headers = (data['headers'] as Map).cast<String, String>();
headers.remove('host');
url = Uri.parse(data['url']);
if (type == _UploadType.multipart) {
field = data['field'];
fileName = data['file-name'];
requestFields = (data['request-fields'] as Map).cast<String, String>();
}
}
}
Raw
google_cloud_storage_native.dart
// in mypod_server project, used to configure google cloud storage access
import 'dart:collection';
import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';
import 'package:crypto/crypto.dart';
import 'package:hex/hex.dart';
import 'package:gcloud/storage.dart';
// import 'package:googleapis/storage/v1.dart' as gcpStorage;
import 'package:googleapis_auth/auth_io.dart' as auth;
// ignore: implementation_imports
import 'package:googleapis_auth/src/crypto/pem.dart';
// ignore: implementation_imports
import 'package:googleapis_auth/src/crypto/rsa_sign.dart';
import 'package:intl/intl.dart';
import 'package:serverpod/serverpod.dart';
// This is a service credentials with storage admin permissions
// (in IAM > service accounts, create a new key for the same service account you were using HMAC keys for)
const _googleClientSecretPath = 'config/google_storage_service_account.json';
/// Concrete implementation of Google Cloud Storage, using native GCP APIs,
/// for use with Serverpod.
class GoogleCloudStorageNative extends CloudStorage {
final String bucket;
final bool public;
late final String publicHost;
late final Storage storage;
late final Map<String, dynamic> _serviceAccount;
/// Creates a new [GoogleCloudStorageNative] object.
static Future<GoogleCloudStorageNative> create({
required Serverpod serverpod,
required String storageId,
required bool public,
required String bucket,
String? publicHost,
}) async {
final instance = GoogleCloudStorageNative._(
serverpod: serverpod,
storageId: storageId,
public: public,
bucket: bucket,
);
await instance._initStorage();
return instance;
}
// Initializes the Google Cloud Storage client.
Future<void> _initStorage() async {
final jsonCredentials = await File(_googleClientSecretPath).readAsString();
var json = jsonDecode(jsonCredentials);
final project = json['project_id'] as String;
final credentials =
auth.ServiceAccountCredentials.fromJson(jsonCredentials);
final client =
await auth.clientViaServiceAccount(credentials, Storage.SCOPES);
storage = Storage(client, project);
final serviceAccount = await File(_googleClientSecretPath).readAsString();
_serviceAccount = jsonDecode(serviceAccount);
}
// Private constructor
GoogleCloudStorageNative._({
required Serverpod serverpod,
required String storageId,
required this.public,
required this.bucket,
String? publicHost,
}) : super(storageId) {
this.publicHost = publicHost ?? 'storage.googleapis.com/$bucket';
}
@override
Future<void> storeFile({
required Session session,
required String path,
required ByteData byteData,
DateTime? expiration,
bool verified = true,
}) async {
await storage
.bucket(bucket)
.writeBytes(path, byteData.buffer.asUint8List());
}
@override
Future<ByteData?> retrieveFile({
required Session session,
required String path,
}) async {
var byteLists = await storage.bucket(bucket).read(path).toList();
var totLength =
byteLists.fold<int>(0, (prev, element) => prev + element.length);
var bytes = Uint8List(totLength);
var offset = 0;
for (var byteList in byteLists) {
bytes.setRange(offset, offset + byteList.length, byteList);
offset += byteList.length;
}
return ByteData.view(bytes.buffer);
}
@override
Future<Uri?> getPublicUrl({
required Session session,
required String path,
}) async {
if (!public) return null;
if (await fileExists(session: session, path: path)) {
return Uri.parse('https://$publicHost/$path');
}
return null;
}
@override
Future<bool> fileExists({
required Session session,
required String path,
}) async {
try {
await storage.bucket(bucket).info(path);
return true;
} catch (e) {
return false;
}
}
@override
Future<void> deleteFile({
required Session session,
required String path,
}) async {
await storage.bucket(bucket).delete(path);
}
// Creates a V4 signed url to allow direct API calls to cloud storage.
String? _createSignedUrl({
required Session session,
required String path,
String? subresource,
int expiration = 604800,
String httpMethod = 'GET',
Map<String, String>? queryParameters,
Map<String, String>? headers,
}) {
if (expiration > 604800) {
session.log(
'Expiration Time can\'t be longer than 604800 seconds (7 days).',
level: LogLevel.error);
return null;
}
final escapedObjectName = Uri.encodeComponent(path);
final canonicalUri = "/$escapedObjectName";
final dateTimeNow = DateTime.now().toUtc();
final requestTimestamp =
DateFormat("yyyyMMdd'T'HHmmss'Z'").format(dateTimeNow);
final datestamp = DateFormat('yyyyMMdd').format(dateTimeNow);
final clientEmail = _serviceAccount['client_email'];
final credentialScope = '$datestamp/auto/storage/goog4_request';
final credential = "$clientEmail/$credentialScope";
headers ??= {};
final host = '$bucket.storage.googleapis.com';
headers['host'] = host;
SplayTreeMap splayTreeMap = SplayTreeMap.from(headers);
headers = Map.from(splayTreeMap);
final canonicalHeaders = headers.entries
.map((entry) =>
'${entry.key.toLowerCase()}:${entry.value.trim().toLowerCase()}\n')
.join();
final signedHeaders = headers.keys.map((k) => k.toLowerCase()).join(';');
queryParameters ??= {};
queryParameters.addAll({
'X-Goog-Algorithm': 'GOOG4-RSA-SHA256',
'X-Goog-Credential': credential,
'X-Goog-Date': requestTimestamp,
'X-Goog-Expires': expiration.toString(),
'X-Goog-SignedHeaders': signedHeaders,
});
if (subresource != null) {
queryParameters[subresource] = '';
}
final canonicalQueryString = queryParameters.entries
.map((entry) =>
'${Uri.encodeComponent(entry.key)}=${Uri.encodeComponent(entry.value)}')
.join('&');
final canonicalRequest = [
httpMethod,
canonicalUri,
canonicalQueryString,
canonicalHeaders,
signedHeaders,
'UNSIGNED-PAYLOAD',
].join("\n");
final canonicalRequestHash =
sha256.convert(utf8.encode(canonicalRequest)).toString();
final stringToSign = [
'GOOG4-RSA-SHA256',
requestTimestamp,
credentialScope,
canonicalRequestHash,
].join('\n');
var pem = _serviceAccount['private_key'];
var rsaKey = keyFromString(pem!);
var signer = RS256Signer(rsaKey);
List<int> stringToSignList = utf8.encode(stringToSign);
var signedRequestBytes = signer.sign(stringToSignList);
var signature = HEX.encode(signedRequestBytes);
final schemeAndHost = 'https://$host';
return '$schemeAndHost$canonicalUri?$canonicalQueryString&x-goog-signature=$signature';
}
@override
Future<String?> createDirectFileUploadDescription({
required Session session,
required String path,
Duration expirationDuration = const Duration(minutes: 10),
}) async {
if (await fileExists(session: session, path: path)) return null;
final headers = {
'content-type': 'application/octet-stream',
'accept': '*/*',
if (public) 'x-goog-acl': 'public-read',
};
final url = _createSignedUrl(
session: session,
path: path,
httpMethod: 'PUT',
expiration: expirationDuration.inSeconds,
headers: headers,
);
var uploadDescriptionData = {
'url': url,
'type': 'binary',
'httpMethod': 'PUT',
'headers': headers,
};
return jsonEncode(uploadDescriptionData);
}
@override
Future<bool> verifyDirectFileUpload({
required Session session,
required String path,
}) async {
return fileExists(session: session, path: path);
}
}
Raw
server.dart
// in mypod_server project, in the main server.dart to configure the different storage buckets
pod.addCloudStorage(await GoogleCloudStorageNative.create(
serverpod: pod,
storageId: 'public',
public: true,
bucket: publicBucket,
));
pod.addCloudStorage(await GoogleCloudStorageNative.create(
serverpod: pod,
storageId: 'private',
public: false,
bucket: privateBucket,
));
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment