penguin/s3-arch-utils

commit 4ac983de0d4e47046865aac152acd3e61a1c75c9

commit 4ac983de0d4e47046865aac152acd3e61a1c75c9
Author: 斟酌 鵬兄 <tgckpg@gmail.com>
Date:   2022-03-30T04:38:18Z

    Added arch_delete_many_aws4.sh
---
 arch_delete_many_aws4.sh | 92 ++++++++++++++++++++++++++++++++++++++++++++++++
 arch_list_aws4.sh        |  8 ++---
 2 files changed, 96 insertions(+), 4 deletions(-)

diff --git a/arch_delete_many_aws4.sh b/arch_delete_many_aws4.sh
new file mode 100755
index 0000000..3374ec1
--- /dev/null
+++ b/arch_delete_many_aws4.sh
@@ -0,0 +1,92 @@
+#!/bin/sh
+# Copyleft https://git.k8s.astropenguin.net/penguin/s3-arch-utils
+
+# ##
+# Reference
+#   https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
+#
+# Usage
+#   arch_delete_aws4.sh fileList.text
+#   cat fileList.txt | arch_delete_aws4.sh -
+#
+# Description
+#   Delete objects from a bucket with provided fileList.txt
+#    * keys must not contain special characters
+#
+# Env vars
+#   ARCH_S3_BUCKET_URL  The bucket url, e.g. my-bucket.s3.us-west-004.backblazeb2.com
+#   ARCH_S3_AUTH        In the format of ACCESS_KEY:SECRET_KEY
+# #
+
+_LIST_SRC=$1
+if [ -z "$_LIST_SRC" ]; then
+	echo "File is not defined, Use \"-\" if you were streaming from stdin"
+	exit 1
+fi
+
+if [ -z "$ARCH_S3_BUCKET_URL" ]; then
+	echo "Env ARCH_S3_BUCKET_URL is required"
+	exit 1
+fi
+
+_TEMP=$( mktemp )
+function __clean_up { rm $_TEMP; }
+trap __clean_up EXIT
+
+echo -n "<Delete>" > $_TEMP
+sed "s/.\+/<Object><Key>\0<\/Key><\/Object>/g" $_LIST_SRC | tr -d '\n' >> $_TEMP
+if [ $? -ne 0 ]; then
+	exit 1
+fi
+echo -n "</Delete>" >> $_TEMP
+
+BUCKET_NAME=$( echo -n $ARCH_S3_BUCKET_URL | cut -d'.' -f1 )
+SERVICE=$( echo -n $ARCH_S3_BUCKET_URL | cut -d'.' -f2 )
+REGION=$( echo -n $ARCH_S3_BUCKET_URL | cut -d'.' -f3 )
+ACCESS_KEY=$( echo -n $ARCH_S3_AUTH | cut -d':' -f1 )
+SECRET_KEY=$( echo -n $ARCH_S3_AUTH | cut -d':' -f2 )
+
+BUCKET_URL=$ARCH_S3_BUCKET_URL
+
+_DATE=$( date -u +"%Y%m%d" )
+_DTIME=$( date -u +"%Y%m%dT%H%M%SZ" )
+_HEADERS="content-md5;host;x-amz-date"
+
+_MD5=$( openssl dgst -md5 -binary $_TEMP | base64 -w0 )
+_SHA=$( sha256sum $_TEMP | cut -d' ' -f1 )
+
+# Canon Request
+_C="POST"
+_C="$_C\n/"
+_C="$_C\ndelete="
+_C="$_C\ncontent-md5:$_MD5"
+_C="$_C\nhost:$BUCKET_URL"
+_C="$_C\nx-amz-date:$_DTIME"
+_C="$_C\n"
+_C="$_C\n$_HEADERS"
+_C="$_C\n$_SHA"
+
+# String to Sign
+_S="AWS4-HMAC-SHA256"
+_S="$_S\n$_DTIME"
+_S="$_S\n$_DATE/$REGION/$SERVICE/aws4_request"
+_S="$_S\n$( echo -ne "$_C" | sha256sum | cut -d' ' -f1 )"
+
+function _HMAC { echo -ne "$2" | openssl dgst -sha256 -hex -mac HMAC -macopt "$1" | cut -d' ' -f2; }
+
+SIG=$( _HMAC "key:AWS4$SECRET_KEY" "$_DATE" )
+SIG=$( _HMAC "hexkey:$SIG" "$REGION" )
+SIG=$( _HMAC "hexkey:$SIG" "$SERVICE" )
+SIG=$( _HMAC "hexkey:$SIG" "aws4_request" )
+SIG=$( _HMAC "hexkey:$SIG" "$_S" )
+
+# BUCKET_URL="127.0.0.1:12345"
+curl -s --data @$_TEMP -XPOST \
+  -H "X-Amz-Date: $_DTIME" \
+  -H "Content-MD5: $_MD5" \
+  -H "Content-Type: application/xml" \
+  -H "X-Amz-Content-SHA256: $_SHA" \
+  -H "Authorization: AWS4-HMAC-SHA256 Credential=$ACCESS_KEY/$_DATE/$REGION/$SERVICE/aws4_request, SignedHeaders=$_HEADERS, Signature=$SIG" \
+  "https://$BUCKET_URL/?delete" \
+  | grep -Eo "<Deleted><Key>[^<]*?</Key>" \
+  | sed "s/^<Deleted><Key>\|<\/Key>//g" | sed "s/^/Deleted \0/g"
diff --git a/arch_list_aws4.sh b/arch_list_aws4.sh
index d8bffe5..cdd3062 100755
--- a/arch_list_aws4.sh
+++ b/arch_list_aws4.sh
@@ -6,10 +6,11 @@
 #   https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
 #
 # Usage
-#   arch_list_aws4.sh path/in/bucket/prefix_
+#   arch_list_aws4.sh [path/in/bucket/prefix_]
 #
 # Description
-#   Print a sorted list of object keys using path/in/bucket/prefix_
+#   Print a list of object keys using path/in/bucket/prefix_
+#   Defaults to print all keys within a bucket if no prefix is defined
 #    * maximum 1000 items, continuation-token is not implemented
 #    * modify _urlencode function to support more special characters
 #
@@ -102,5 +103,4 @@ curl -s -XGET \
   -H "Authorization: AWS4-HMAC-SHA256 Credential=$ACCESS_KEY/$_DATE/$REGION/$SERVICE/aws4_request, SignedHeaders=$_HEADERS, Signature=$SIG" \
   "https://$BUCKET_URL/$_PATH?$QSTR" \
   | grep -Eo "<Key>[^<]*?</Key>" \
-  | sed "s/^<Key>\|<\/Key>//g" \
-  | sort
+  | sed "s/^<Key>\|<\/Key>//g"