Files
simapi-cj/src/helpers/SimApiStorage.cj
T
xrain 67f7c16a3f feat: SimApiStorage 增加 deleteFiles 批量删除(S3 DeleteObjects)
- POST /{bucket}?delete + XML 一次删除多个对象(Minio RemoveObjects 同款接口)
- MinIO 强制要求 Content-Md5 头(缺失返回 MissingContentMD5),MD5 经 stdx 计算 + Base64
- 已用 oss.anymemory.cn 实测:批量删除后访问对象返回 404
2026-08-17 00:22:08 +08:00

537 lines
21 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/*
* Copyright (c) 2025 SimcuTeam. All rights reserved.
* 移植自 C# 项目 SimApiE:\simcu\simapi-net),遵循 MIT 许可证。
* Helpers/SimApiStorageS3/MinIO 兼容存储助手。
*
* 仓颉生态暂无 Minio SDK,此处自实现 AWS Signature V4HMAC-SHA256 基于 stdx SHA256):
* - 预签名 URLPUT 上传 / GET 下载)
* - 直接 PUT 上传(对象 + 建桶)
* - HEAD 检测桶是否存在
* 与 C# SimApiStorageMinio SDK)的公开方法语义对齐:
* GetUploadUrl / GetDownloadUrl / UploadFile / FullUrl / GetUrl / GetPath
*/
package simapi.helpers
import std.collection.*
import std.time.*
import stdx.crypto.digest.*
import stdx.encoding.base64.*
import stdx.encoding.hex.*
import stdx.net.tls.*
import stdx.net.tls.common.*
import soulsoft_net_http.{ByteArrayContent, HttpClient, HttpRequestMessage}
import soulsoft_net_http.{HttpMethod as NetHttpMethod}
import soulsoft_web_http.*
import simapi.configurations.*
/**
* 上传预签名 URL 响应(对齐 C# record GetUploadUrlResponse(UploadUrl, DownloadUrl, Path))。
*/
public class GetUploadUrlResponse {
public var uploadUrl: String = ""
public var downloadUrl: String = ""
public var path: String = ""
public init() {}
public init(uploadUrl: String, downloadUrl: String, path: String) {
this.uploadUrl = uploadUrl
this.downloadUrl = downloadUrl
this.path = path
}
}
/**
* S3/MinIO 存储助手(对齐 C# Helpers/SimApiStorage)。
*
* 说明:
* - 注册为 Scoped 以注入 IHttpContextAccessorsoulsoft DI 禁止 singleton 消费 scoped 服务);
* 桶的检测/创建由静态守卫保证整个进程只执行一次(对齐 C# 构造函数中 BucketExists+MakeBucket)。
* - fullUrl/getUrl 的 "~/" 分支依赖当前请求上下文(对齐 C# IHttpContextAccessor)。
*/
public class SimApiStorage {
private static var _bucketEnsured: Bool = false
private let _endpoint: String
private let _serveUrl: String
private let _bucket: String
private let _accessKey: String
private let _secretKey: String
private let _useSsl: Bool
private let _host: String
private let _region: String = "us-east-1"
private let _httpContextAccessor: IHttpContextAccessor
public init(options: SimApiOptions, httpContextAccessor: IHttpContextAccessor) {
let storage = options.simApiStorageOptions
if (storage.endpoint.isEmpty() || storage.serveUrl.isEmpty() || storage.bucket.isEmpty()) {
throw Exception("SimApiStorage: Endpoint/ServeUrl/Bucket 不能为空")
}
var useSsl = false
var host = ""
if (storage.endpoint.startsWith("http://")) {
host = storage.endpoint["http://".size..]
} else if (storage.endpoint.startsWith("https://")) {
useSsl = true
host = storage.endpoint["https://".size..]
} else {
throw Exception("SimApiStorage: Error Endpoint")
}
if (storage.serveUrl.endsWith("/")) {
throw Exception("SimApiStorage: ServeUrl must not end with /")
}
_endpoint = storage.endpoint
_serveUrl = storage.serveUrl
_bucket = storage.bucket
_accessKey = storage.accessKey
_secretKey = storage.secretKey
_useSsl = useSsl
_host = host
_httpContextAccessor = httpContextAccessor
// 桶不存在则创建(对齐 C# BucketExists + MakeBucket;静态守卫保证只执行一次)
ensureBucketOnce()
}
/**
* 获取上传预签名 URL(对齐 C# GetUploadUrl,默认 7200 秒)。
*/
public func getUploadUrl(path: String, expire!: Int64 = 7200): GetUploadUrlResponse {
checkPath(path)
let obj = trimLeadingSlash(path)
let uploadUrl = presign("PUT", obj, expire)
GetUploadUrlResponse(uploadUrl, "${_serveUrl}${path}", path)
}
/**
* 获取下载预签名 URL(对齐 C# GetDownloadUrl,默认 600 秒)。
*/
public func getDownloadUrl(path: String, expire!: Int64 = 600): String {
checkPath(path)
let obj = trimLeadingSlash(path)
presign("GET", obj, expire)
}
/**
* 直接上传文件(对齐 C# UploadFiledata 为文件字节)。
*/
public func uploadFile(path: String, data: Array<Byte>, contentType!: String = "image/png"): Unit {
checkPath(path)
let obj = trimLeadingSlash(path)
putObject(obj, data, contentType)
}
/**
* 批量删除对象(对齐 C# Minio RemoveObjectsAsyncS3 原生 DeleteObjects 接口:
* POST /{bucket}?delete,一次请求删除多个对象,无需逐个删除)。
* @param paths 对象路径数组(每个须以 / 开头)。
*/
public func deleteFiles(paths: Array<String>): Unit {
if (paths.isEmpty()) {
return
}
for (p in paths) {
checkPath(p)
}
var sb = StringBuilder()
sb.append("<Delete xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">")
for (p in paths) {
sb.append("<Object><Key>${xmlEscape(trimLeadingSlash(p))}</Key></Object>")
}
sb.append("<Quiet>true</Quiet></Delete>")
deleteObjects(sb.toString())
}
/**
* 使用 path 获取完整的访问 URL(对齐 C# FullUrl)。
*/
public func fullUrl(path: ?String): ?String {
if (let Some(p) <- path) {
if (p.isEmpty() || p.startsWith("http://") || p.startsWith("https://")) {
return Some(p)
}
if (!(p.startsWith("/") || p.startsWith("~/"))) {
return Some(p)
}
if (p.startsWith("~/")) {
return Some("${requestBaseUrl()}${p[1..]}")
}
return Some("${_serveUrl}${p}")
}
None
}
/**
* 获取一个 Path 的访问 URL(对齐 C# GetUrl)。
*/
public func getUrl(path: ?String): ?String {
if (let Some(p) <- path) {
if (p.isEmpty()) {
return Some(p)
}
if (p.startsWith("~/")) {
return Some("${requestBaseUrl()}${p[1..]}")
}
if (p.startsWith("/")) {
return Some("${_serveUrl}${p}")
}
return Some(p)
}
None
}
/**
* 从 URL 中获取相对路径(对齐 C# GetPath;去掉 Endpoint/Bucket 或 ServeUrl 前缀)。
*/
public func getPath(url: ?String): ?String {
if (let Some(u) <- url) {
var r = u
let prefix = "${_endpoint}/${_bucket}"
if (r.startsWith(prefix)) {
r = r[prefix.size..]
}
if (r.startsWith(_serveUrl)) {
r = r[_serveUrl.size..]
}
return Some(r)
}
None
}
// ===== 私有:SigV4 签名与 S3 请求 =====
/// 预签名 URL(对齐 Minio PresignedPutObject / PresignedGetObject
private func presign(method: String, object: String, expireSeconds: Int64): String {
let now = DateTime.nowUTC()
let amzDate = formatAmzDate(now)
let dateStamp = amzDate[0..8]
let canonicalUri = "/${_bucket}/${uriEncode(object, false)}"
let credential = "${_accessKey}/${dateStamp}/${_region}/s3/aws4_request"
// 五个 X-Amz-* 参数按字典序排列(A<C<D<E<S
let canonicalQuery = "X-Amz-Algorithm=${uriEncode("AWS4-HMAC-SHA256", true)}" +
"&X-Amz-Credential=${uriEncode(credential, true)}" +
"&X-Amz-Date=${amzDate}" +
"&X-Amz-Expires=${expireSeconds}" +
"&X-Amz-SignedHeaders=host"
let canonicalHeaders = "host:${_host}\n"
let signedHeaders = "host"
let canonicalRequest = "${method}\n${canonicalUri}\n${canonicalQuery}\n${canonicalHeaders}\n${signedHeaders}\nUNSIGNED-PAYLOAD"
let stringToSign = "AWS4-HMAC-SHA256\n${amzDate}\n${dateStamp}/${_region}/s3/aws4_request\n" +
"${toHexString(sha256Bytes(canonicalRequest.toArray()))}"
let signature = toHexString(hmacSha256(buildSigningKey(dateStamp), stringToSign.toArray()))
"${_endpoint}${canonicalUri}?${canonicalQuery}&X-Amz-Signature=${signature}"
}
/// 直接 PUT 上传对象(Authorization 头签名,对齐 Minio PutObject
private func putObject(object: String, data: Array<Byte>, contentType: String): Unit {
let now = DateTime.nowUTC()
let amzDate = formatAmzDate(now)
let dateStamp = amzDate[0..8]
let payloadHash = toHexString(sha256Bytes(data))
let canonicalUri = "/${_bucket}/${uriEncode(object, false)}"
let canonicalHeaders = "host:${_host}\nx-amz-content-sha256:${payloadHash}\nx-amz-date:${amzDate}\n"
let signedHeaders = "host;x-amz-content-sha256;x-amz-date"
let canonicalRequest = "PUT\n${canonicalUri}\n\n${canonicalHeaders}\n${signedHeaders}\n${payloadHash}"
let stringToSign = "AWS4-HMAC-SHA256\n${amzDate}\n${dateStamp}/${_region}/s3/aws4_request\n" +
"${toHexString(sha256Bytes(canonicalRequest.toArray()))}"
let authorization = buildAuthorization(dateStamp, signedHeaders,
toHexString(hmacSha256(buildSigningKey(dateStamp), stringToSign.toArray())))
let client = createClient()
try {
let request = HttpRequestMessage(NetHttpMethod.Put, "${_endpoint}${canonicalUri}")
request.headers.add("x-amz-content-sha256", payloadHash)
request.headers.add("x-amz-date", amzDate)
request.headers.add("Authorization", authorization)
let content = ByteArrayContent(data)
content.headers.add("Content-Type", contentType)
request.content = content
let response = client.send(request)
try {
SimApiError.errorWhenFalse(response.isSuccessStatusCode, code: response.statusCode,
message: "SimApiStorage 上传失败: HTTP ${response.statusCode}")
} finally {
response.close()
}
} finally {
client.close()
}
}
/// HEAD 检测桶是否存在(对齐 Minio BucketExistsAsync
private func bucketExists(): Bool {
let now = DateTime.nowUTC()
let amzDate = formatAmzDate(now)
let dateStamp = amzDate[0..8]
let payloadHash = toHexString(sha256Bytes(Array<Byte>(0, repeat: 0)))
let canonicalUri = "/${_bucket}"
let canonicalHeaders = "host:${_host}\nx-amz-content-sha256:${payloadHash}\nx-amz-date:${amzDate}\n"
let signedHeaders = "host;x-amz-content-sha256;x-amz-date"
let canonicalRequest = "HEAD\n${canonicalUri}\n\n${canonicalHeaders}\n${signedHeaders}\n${payloadHash}"
let stringToSign = "AWS4-HMAC-SHA256\n${amzDate}\n${dateStamp}/${_region}/s3/aws4_request\n" +
"${toHexString(sha256Bytes(canonicalRequest.toArray()))}"
let authorization = buildAuthorization(dateStamp, signedHeaders,
toHexString(hmacSha256(buildSigningKey(dateStamp), stringToSign.toArray())))
let client = createClient()
try {
let request = HttpRequestMessage(NetHttpMethod.Head, "${_endpoint}${canonicalUri}")
request.headers.add("x-amz-content-sha256", payloadHash)
request.headers.add("x-amz-date", amzDate)
request.headers.add("Authorization", authorization)
let response = client.send(request)
try {
response.statusCode == 200
} finally {
response.close()
}
} finally {
client.close()
}
}
/// 创建桶(对齐 Minio MakeBucketAsync
private func makeBucket(): Unit {
let now = DateTime.nowUTC()
let amzDate = formatAmzDate(now)
let dateStamp = amzDate[0..8]
let payloadHash = toHexString(sha256Bytes(Array<Byte>(0, repeat: 0)))
let canonicalUri = "/${_bucket}"
let canonicalHeaders = "host:${_host}\nx-amz-content-sha256:${payloadHash}\nx-amz-date:${amzDate}\n"
let signedHeaders = "host;x-amz-content-sha256;x-amz-date"
let canonicalRequest = "PUT\n${canonicalUri}\n\n${canonicalHeaders}\n${signedHeaders}\n${payloadHash}"
let stringToSign = "AWS4-HMAC-SHA256\n${amzDate}\n${dateStamp}/${_region}/s3/aws4_request\n" +
"${toHexString(sha256Bytes(canonicalRequest.toArray()))}"
let authorization = buildAuthorization(dateStamp, signedHeaders,
toHexString(hmacSha256(buildSigningKey(dateStamp), stringToSign.toArray())))
let client = createClient()
try {
let request = HttpRequestMessage(NetHttpMethod.Put, "${_endpoint}${canonicalUri}")
request.headers.add("x-amz-content-sha256", payloadHash)
request.headers.add("x-amz-date", amzDate)
request.headers.add("Authorization", authorization)
request.content = ByteArrayContent(Array<Byte>(0, repeat: 0))
let response = client.send(request)
try {
SimApiError.errorWhenFalse(response.isSuccessStatusCode, code: response.statusCode,
message: "SimApiStorage 创建桶失败: HTTP ${response.statusCode}")
} finally {
response.close()
}
} finally {
client.close()
}
}
private func ensureBucketOnce(): Unit {
if (!SimApiStorage._bucketEnsured) {
if (!bucketExists()) {
makeBucket()
}
SimApiStorage._bucketEnsured = true
}
}
/// 批量删除请求(S3 DeleteObjectsPOST /{bucket}?delete + XML
private func deleteObjects(xml: String): Unit {
let now = DateTime.nowUTC()
let amzDate = formatAmzDate(now)
let dateStamp = amzDate[0..8]
let payloadHash = toHexString(sha256Bytes(xml.toArray()))
let canonicalUri = "/${_bucket}"
let canonicalQuery = "delete="
let canonicalHeaders = "host:${_host}\nx-amz-content-sha256:${payloadHash}\nx-amz-date:${amzDate}\n"
let signedHeaders = "host;x-amz-content-sha256;x-amz-date"
let canonicalRequest = "POST\n${canonicalUri}\n${canonicalQuery}\n${canonicalHeaders}\n${signedHeaders}\n${payloadHash}"
let stringToSign = "AWS4-HMAC-SHA256\n${amzDate}\n${dateStamp}/${_region}/s3/aws4_request\n" +
"${toHexString(sha256Bytes(canonicalRequest.toArray()))}"
let authorization = buildAuthorization(dateStamp, signedHeaders,
toHexString(hmacSha256(buildSigningKey(dateStamp), stringToSign.toArray())))
let client = createClient()
try {
let request = HttpRequestMessage(NetHttpMethod.Post, "${_endpoint}${canonicalUri}?${canonicalQuery}")
request.headers.add("x-amz-content-sha256", payloadHash)
request.headers.add("x-amz-date", amzDate)
request.headers.add("Authorization", authorization)
let contentMd5 = md5Base64(xml.toArray())
let content = ByteArrayContent(xml.toArray())
content.headers.add("Content-Type", "application/xml")
// MinIO 的 DeleteObjects 强制要求 Content-Md5(缺失返回 MissingContentMD5
content.headers.add("Content-Md5", contentMd5)
request.content = content
let response = client.send(request)
try {
SimApiError.errorWhenFalse(response.isSuccessStatusCode, code: response.statusCode,
message: "SimApiStorage 批量删除失败: HTTP ${response.statusCode}")
} finally {
response.close()
}
} finally {
client.close()
}
}
private func buildAuthorization(dateStamp: String, signedHeaders: String, signature: String): String {
"AWS4-HMAC-SHA256 Credential=${_accessKey}/${dateStamp}/${_region}/s3/aws4_request, " +
"SignedHeaders=${signedHeaders}, Signature=${signature}"
}
private func buildSigningKey(dateStamp: String): Array<Byte> {
let kDate = hmacSha256("AWS4${_secretKey}".toArray(), dateStamp.toArray())
let kRegion = hmacSha256(kDate, _region.toArray())
let kService = hmacSha256(kRegion, "s3".toArray())
hmacSha256(kService, "aws4_request".toArray())
}
private func createClient(): HttpClient {
HttpClient.create { builder =>
builder.noProxy()
if (_useSsl) {
var tls = TlsClientConfig()
tls.verifyMode = CertificateVerifyMode.TrustAll
let host = extractHost(_endpoint)
if (!host.isEmpty()) {
tls.serverName = Some(host)
}
builder.tlsConfig(tls)
}
}
}
private func checkPath(path: String): Unit {
if (!path.startsWith("/")) {
throw Exception("path must start with /")
}
}
private func trimLeadingSlash(path: String): String {
if (path.startsWith("/")) {
path[1..]
} else {
path
}
}
/// 当前请求基础地址(scheme://host);无请求上下文时返回空串
private func requestBaseUrl(): String {
if (let Some(ctx) <- _httpContextAccessor.context) {
return "${ctx.request.scheme}://${ctx.request.host}"
}
""
}
private static func extractHost(fullUrl: String): String {
match (fullUrl.indexOf("://")) {
case Some(i) =>
let rest = fullUrl[i + 3..]
let slash = rest.indexOf("/") ?? rest.size
let q = rest.indexOf("?") ?? rest.size
let end = if (slash < q) { slash } else { q }
rest[0..end]
case None => ""
}
}
private static func formatAmzDate(dt: DateTime): String {
"${dt.year}${pad2(dt.monthValue)}${pad2(dt.dayOfMonth)}T${pad2(dt.hour)}${pad2(dt.minute)}${pad2(dt.second)}Z"
}
private static func pad2(v: Int64): String {
if (v < 10) {
"0${v}"
} else {
"${v}"
}
}
/// XML 特殊字符转义(对象 Key 中可能含 & < > " '
private static func xmlEscape(s: String): String {
var sb = StringBuilder()
for (c in s.runes()) {
match (c) {
case '&' => sb.append("&amp;")
case '<' => sb.append("&lt;")
case '>' => sb.append("&gt;")
case '"' => sb.append("&quot;")
case '\'' => sb.append("&apos;")
case _ => sb.append(c)
}
}
sb.toString()
}
/// RFC 3986 百分号编码(encodeSlash=false 时保留 '/'
private static func uriEncode(s: String, encodeSlash: Bool): String {
let bytes = s.toArray()
let hex = "0123456789ABCDEF"
var sb = StringBuilder()
for (b in bytes) {
let u = toU8(b)
let unreserved = (u >= 0x41u8 && u <= 0x5Au8) || (u >= 0x61u8 && u <= 0x7Au8) ||
(u >= 0x30u8 && u <= 0x39u8) || u == 0x2Du8 || u == 0x5Fu8 || u == 0x2Eu8 || u == 0x7Eu8
if (unreserved || (u == 0x2Fu8 && !encodeSlash)) {
sb.append(Rune(UInt32(u)))
} else {
sb.append("%")
sb.append(Rune(UInt32(hex[Int64((u >> 4u8) & 0x0Fu8)])))
sb.append(Rune(UInt32(hex[Int64(u & 0x0Fu8)])))
}
}
sb.toString()
}
private static func toU8(b: Byte): UInt8 {
if (b < 0) {
UInt8(Int64(b) + 256)
} else {
UInt8(Int64(b))
}
}
/// HMAC-SHA256(自实现,基于 stdx SHA256stdx 静态库无现成 HMAC 封装)
private static func hmacSha256(key: Array<Byte>, data: Array<Byte>): Array<Byte> {
var k = key
if (k.size > 64) {
k = sha256Bytes(k)
}
var keyPadded = Array<Byte>(64, repeat: 0)
for (i in 0..k.size) {
keyPadded[i] = k[i]
}
var inner = ArrayList<Byte>()
for (i in 0..64) {
inner.add(bxor(keyPadded[i], 0x36u8))
}
for (b in data) {
inner.add(b)
}
let innerHash = sha256Bytes(inner.toArray())
var outer = ArrayList<Byte>()
for (i in 0..64) {
outer.add(bxor(keyPadded[i], 0x5Cu8))
}
for (b in innerHash) {
outer.add(b)
}
sha256Bytes(outer.toArray())
}
@OverflowWrapping
private static func bxor(a: Byte, mask: UInt8): Byte {
let ua = toU8(a)
let r: Byte = ua ^ mask
r
}
private static func sha256Bytes(data: Array<Byte>): Array<Byte> {
let sha = SHA256()
sha.write(data)
sha.finish()
}
/// MD5 → Base64Content-Md5 头,S3 DeleteObjects 必需)
private static func md5Base64(data: Array<Byte>): String {
let md = MD5()
md.write(data)
toBase64String(md.finish())
}
}