From 22d73690dfaa9e3abafd8c2dfa4ead37a7a8f801 Mon Sep 17 00:00:00 2001 From: Sven Ulland Date: Mon, 1 Jun 2026 11:03:57 +0200 Subject: [PATCH 1/2] s3: map --tags onto object-creating requests The aws s3 cp/mv/sync commands already let users set most write-time object properties (storage class, metadata, SSE, ...) but not tags, forcing a separate PutObjectTagging pass after the transfer. That pass doubles the request count, and any object whose tagging step is skipped or fails stays untagged, so tag-based lifecycle rules never match it. Teach RequestParamsMapper to build the Tagging member of PutObject and CreateMultipartUpload from the repeatable --tags key/value pairs, URL-encoding them into the x-amz-tagging query-string form. s3transfer forwards these extra args to CreateMultipartUpload and drops them from the per-part requests, so the tag set is applied to the object exactly once regardless of size. This commit only adds the mapping; no CLI parameter feeds it yet, so it is inert on its own. --- awscli/customizations/s3/utils.py | 11 +++++++++ tests/unit/customizations/s3/test_utils.py | 27 ++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/awscli/customizations/s3/utils.py b/awscli/customizations/s3/utils.py index 9051ef69bc09..1e502d027d13 100644 --- a/awscli/customizations/s3/utils.py +++ b/awscli/customizations/s3/utils.py @@ -19,6 +19,7 @@ import re import time from collections import namedtuple, deque +from urllib.parse import urlencode from dateutil.parser import parse from dateutil.tz import tzlocal, tzutc @@ -475,6 +476,7 @@ def map_put_object_params(cls, request_params, cli_params): """Map CLI params to PutObject request params""" cls._set_general_object_params(request_params, cli_params) cls._set_metadata_params(request_params, cli_params) + cls._set_tagging_params(request_params, cli_params) cls._set_sse_request_params(request_params, cli_params) cls._set_sse_c_request_params(request_params, cli_params) cls._set_request_payer_param(request_params, cli_params) @@ -526,6 +528,7 @@ def map_create_multipart_upload_params(cls, request_params, cli_params): cls._set_sse_request_params(request_params, cli_params) cls._set_sse_c_request_params(request_params, cli_params) cls._set_metadata_params(request_params, cli_params) + cls._set_tagging_params(request_params, cli_params) cls._set_request_payer_param(request_params, cli_params) @classmethod @@ -615,6 +618,14 @@ def _set_metadata_params(cls, request_params, cli_params): if cli_params.get('metadata'): request_params['Metadata'] = cli_params['metadata'] + @classmethod + def _set_tagging_params(cls, request_params, cli_params): + # Tagging is a URL-encoded query string built from the --tags pairs. + if cli_params.get('tags'): + request_params['Tagging'] = urlencode( + [(key, value) for key, value in cli_params['tags']] + ) + @classmethod def _auto_populate_metadata_directive(cls, request_params): if request_params.get('Metadata') and \ diff --git a/tests/unit/customizations/s3/test_utils.py b/tests/unit/customizations/s3/test_utils.py index 7a29b8f01b9d..ccce2c10e14a 100644 --- a/tests/unit/customizations/s3/test_utils.py +++ b/tests/unit/customizations/s3/test_utils.py @@ -704,6 +704,33 @@ def test_upload_part_copy(self): 'SSECustomerKey': 'my-sse-c-key'}) +class TestRequestParamsMapperTagging(unittest.TestCase): + def setUp(self): + self.cli_params = {'tags': [['key1', 'value1'], ['key2', 'value2']]} + + def test_put_object(self): + params = {} + RequestParamsMapper.map_put_object_params(params, self.cli_params) + self.assertEqual(params, {'Tagging': 'key1=value1&key2=value2'}) + + def test_create_multipart_upload(self): + params = {} + RequestParamsMapper.map_create_multipart_upload_params( + params, self.cli_params) + self.assertEqual(params, {'Tagging': 'key1=value1&key2=value2'}) + + def test_tag_values_are_url_encoded(self): + params = {} + cli_params = {'tags': [['key', 'a value/with=specials']]} + RequestParamsMapper.map_put_object_params(params, cli_params) + self.assertEqual(params, {'Tagging': 'key=a+value%2Fwith%3Dspecials'}) + + def test_no_tags_param_is_a_noop(self): + params = {} + RequestParamsMapper.map_put_object_params(params, {}) + self.assertEqual(params, {}) + + class TestRequestParamsMapperChecksumAlgorithm: @pytest.fixture def cli_params(self): From 0ec4ffae225ac3ae0417e076a89c14ca3c433166 Mon Sep 17 00:00:00 2001 From: Sven Ulland Date: Mon, 1 Jun 2026 11:04:08 +0200 Subject: [PATCH 2/2] s3: add a --tags parameter to cp, mv, and sync Expose the object tagging support wired into RequestParamsMapper as a --tags parameter on the transfer commands. It takes repeatable --tags pairs, matching the s3 mb --tags syntax, and applies them to each object as it is created. Like every other write-time property, it only affects objects that are actually uploaded; in a sync, files that have not changed are not retagged. This matches the long-standing behaviour of --metadata, --storage-class and --acl and is called out in the parameter's help text. S3-to-S3 copies remain governed by --copy-props and are intentionally out of scope for this change. --- .../next-release/enhancement-s3-30038.json | 5 +++ awscli/customizations/s3/subcommands.py | 26 +++++++++++-- awscli/examples/s3/cp.rst | 9 +++++ awscli/examples/s3/mv.rst | 15 +++++++- awscli/examples/s3/sync.rst | 13 +++++++ tests/functional/s3/test_cp_command.py | 37 +++++++++++++++++++ tests/functional/s3/test_mv_command.py | 13 +++++++ tests/functional/s3/test_sync_command.py | 16 ++++++++ 8 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 .changes/next-release/enhancement-s3-30038.json diff --git a/.changes/next-release/enhancement-s3-30038.json b/.changes/next-release/enhancement-s3-30038.json new file mode 100644 index 000000000000..5b0dde54aff3 --- /dev/null +++ b/.changes/next-release/enhancement-s3-30038.json @@ -0,0 +1,5 @@ +{ + "type": "enhancement", + "category": "``s3``", + "description": "Add a ``--tags`` parameter to the ``s3 cp``, ``s3 mv``, and ``s3 sync`` commands to set object tags as objects are uploaded to S3." +} diff --git a/awscli/customizations/s3/subcommands.py b/awscli/customizations/s3/subcommands.py index bd3f60f16479..19d81dbf3bb9 100644 --- a/awscli/customizations/s3/subcommands.py +++ b/awscli/customizations/s3/subcommands.py @@ -328,6 +328,25 @@ } +OBJECT_TAGS = { + 'name': 'tags', + 'synopsis': '--tags ', + 'action': 'append', + 'nargs': 2, + 'help_text': ( + 'Tags to apply to objects uploaded to S3 by this operation, in ' + 'the format ``--tags key value``. Specify this parameter multiple ' + 'times, once for each tag. The tags are applied as the objects are ' + 'created, so no separate ``put-object-tagging`` call is made. They ' + 'are applied to every object which is part of this request. In a ' + "sync, this means that files which haven't changed won't receive " + 'the tags. This only affects objects uploaded to S3; it has no ' + 'effect on downloads, and copies between two S3 locations are ' + 'governed by ``--copy-props`` instead.' + ) +} + + METADATA_DIRECTIVE = { 'name': 'metadata-directive', 'choices': ['COPY', 'REPLACE'], 'help_text': ( @@ -528,9 +547,10 @@ SSE_C_COPY_SOURCE_KEY, STORAGE_CLASS, GRANTS, WEBSITE_REDIRECT, CONTENT_TYPE, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, - EXPIRES, SOURCE_REGION, ONLY_SHOW_ERRORS, NO_PROGRESS, - PAGE_SIZE, IGNORE_GLACIER_WARNINGS, FORCE_GLACIER_TRANSFER, - REQUEST_PAYER, CHECKSUM_MODE, CHECKSUM_ALGORITHM] + EXPIRES, OBJECT_TAGS, SOURCE_REGION, ONLY_SHOW_ERRORS, + NO_PROGRESS, PAGE_SIZE, IGNORE_GLACIER_WARNINGS, + FORCE_GLACIER_TRANSFER, REQUEST_PAYER, CHECKSUM_MODE, + CHECKSUM_ALGORITHM] def get_client(session, region, endpoint_url, verify, config=None): diff --git a/awscli/examples/s3/cp.rst b/awscli/examples/s3/cp.rst index eb2529c28d35..cec5a5b3c700 100644 --- a/awscli/examples/s3/cp.rst +++ b/awscli/examples/s3/cp.rst @@ -200,3 +200,12 @@ The following ``cp`` command downloads a single object (``mykey``) from the acce Output:: download: s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/mykey to mydoc.txt + +**Example 16: Uploading an object and applying tags** + +The following ``cp`` command uploads a single file to a specified bucket and key and applies two tags to the object as +it is created, so no separate ``put-object-tagging`` call is needed:: + + aws s3 cp test.txt s3://amzn-s3-demo-bucket/test.txt \ + --tags Key1 Value1 \ + --tags Key2 Value2 diff --git a/awscli/examples/s3/mv.rst b/awscli/examples/s3/mv.rst index 836d5d0fcae0..1deb00a483b0 100644 --- a/awscli/examples/s3/mv.rst +++ b/awscli/examples/s3/mv.rst @@ -112,4 +112,17 @@ The following ``mv`` command moves a single file named ``mydoc.txt`` to the acce Output:: - move: mydoc.txt to s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/mykey \ No newline at end of file + move: mydoc.txt to s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/mykey + +**Example 11: Move a file to S3 and apply tags** + +The following ``mv`` command moves a single file to a specified bucket and key and applies two tags to the object as it +is uploaded:: + + aws s3 mv file.txt s3://amzn-s3-demo-bucket/file.txt \ + --tags Key1 Value1 \ + --tags Key2 Value2 + +Output:: + + move: file.txt to s3://amzn-s3-demo-bucket/file.txt \ No newline at end of file diff --git a/awscli/examples/s3/sync.rst b/awscli/examples/s3/sync.rst index bee26f6684fc..b27fa2ab7748 100644 --- a/awscli/examples/s3/sync.rst +++ b/awscli/examples/s3/sync.rst @@ -117,3 +117,16 @@ Output:: upload: test.txt to s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/test.txt upload: test2.txt to s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/test2.txt + +**Example 9: Sync to S3 and apply tags** + +The following ``sync`` command syncs the current directory to a specified bucket and applies two tags to each object as +it is uploaded. Because tags are applied only as objects are uploaded, files that are already in sync are not retagged:: + + aws s3 sync . s3://amzn-s3-demo-bucket \ + --tags Key1 Value1 \ + --tags Key2 Value2 + +Output:: + + upload: test.txt to s3://amzn-s3-demo-bucket/test.txt diff --git a/tests/functional/s3/test_cp_command.py b/tests/functional/s3/test_cp_command.py index 643a5f734a45..866cd9513108 100644 --- a/tests/functional/s3/test_cp_command.py +++ b/tests/functional/s3/test_cp_command.py @@ -121,6 +121,43 @@ def test_upload_standard_ia(self): self.assertEqual(args['Bucket'], 'bucket') self.assertEqual(args['StorageClass'], 'STANDARD_IA') + def test_upload_with_tagging(self): + full_path = self.files.create_file('foo.txt', 'mycontent') + cmdline = ( + '%s %s s3://bucket/key.txt --tags key1 value1 --tags key2 value2' % + (self.prefix, full_path)) + self.parsed_responses = \ + [{'ETag': '"c8afdb36c52cf4727836669019e69222"'}] + self.run_cmd(cmdline, expected_rc=0) + self.assertEqual(len(self.operations_called), 1, + self.operations_called) + self.assertEqual(self.operations_called[0][0].name, 'PutObject') + self.assertEqual(self.operations_called[0][1]['Tagging'], + 'key1=value1&key2=value2') + + def test_multipart_upload_with_tagging(self): + # The tag set must be applied once, on CreateMultipartUpload, and must + # not leak onto the individual UploadPart requests (S3 rejects it + # there). + full_path = self.files.create_file('foo.txt', 'a' * 10 * (1024 ** 2)) + self.parsed_responses = [ + {'UploadId': 'foo'}, + {'ETag': '"foo-1"'}, + {'ETag': '"foo-2"'}, + {} + ] + cmdline = ('%s %s s3://bucket/key.txt' + ' --tags key1 value1' % (self.prefix, full_path)) + self.run_cmd(cmdline, expected_rc=0) + operations = [op[0].name for op in self.operations_called] + self.assertEqual( + operations, + ['CreateMultipartUpload', 'UploadPart', 'UploadPart', + 'CompleteMultipartUpload']) + self.assertEqual(self.operations_called[0][1]['Tagging'], + 'key1=value1') + self.assertNotIn('Tagging', self.operations_called[1][1]) + def test_upload_onezone_ia(self): full_path = self.files.create_file('foo.txt', 'mycontent') cmdline = ('%s %s s3://bucket/key.txt --storage-class ONEZONE_IA' % diff --git a/tests/functional/s3/test_mv_command.py b/tests/functional/s3/test_mv_command.py index 947122cfa7cc..49f36ca651d1 100644 --- a/tests/functional/s3/test_mv_command.py +++ b/tests/functional/s3/test_mv_command.py @@ -48,6 +48,19 @@ def test_website_redirect_ignore_paramfile(self): 'http://someserver' ) + def test_upload_with_tagging(self): + full_path = self.files.create_file('foo.txt', 'mycontent') + cmdline = ( + '%s %s s3://bucket/key.txt --tags key1 value1 --tags key2 value2' % + (self.prefix, full_path)) + self.parsed_responses = [{'ETag': '"c8afdb36c52cf4727836669019e69222"'}] + self.run_cmd(cmdline, expected_rc=0) + self.assertEqual(self.operations_called[0][0].name, 'PutObject') + self.assertEqual( + self.operations_called[0][1]['Tagging'], + 'key1=value1&key2=value2' + ) + def test_metadata_directive_copy(self): self.parsed_responses = [ { diff --git a/tests/functional/s3/test_sync_command.py b/tests/functional/s3/test_sync_command.py index e2b9ab40e30e..28bf83402b7d 100644 --- a/tests/functional/s3/test_sync_command.py +++ b/tests/functional/s3/test_sync_command.py @@ -43,6 +43,22 @@ def test_website_redirect_ignore_paramfile(self): 'http://someserver' ) + def test_upload_with_tagging(self): + self.files.create_file('foo.txt', 'mycontent') + cmdline = ( + '%s %s s3://bucket/key.txt --tags key1 value1 --tags key2 value2' % + (self.prefix, self.files.rootdir)) + self.parsed_responses = [ + {"CommonPrefixes": [], "Contents": []}, + {'ETag': '"c8afdb36c52cf4727836669019e69222"'} + ] + self.run_cmd(cmdline, expected_rc=0) + self.assertEqual(self.operations_called[1][0].name, 'PutObject') + self.assertEqual( + self.operations_called[1][1]['Tagging'], + 'key1=value1&key2=value2' + ) + def test_no_recursive_option(self): cmdline = '. s3://mybucket --recursive' # Return code will be 2 for invalid parameter ``--recursive``