Add tagging support for the s3 cp, mv, and sync commands

.changes/next-release/enhancement-s3-30038.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "``s3``",
+  "description": "Add a ``--tags`` parameter to the ``s3 cp``, ``s3 mv``, and ``s3 sync`` commands to set object tags as objects are uploaded to S3."
+}

awscli/customizations/s3/subcommands.py

@@ -328,6 +328,25 @@
 }
 
 
+OBJECT_TAGS = {
+    'name': 'tags',
+    'synopsis': '--tags <key> <value>',
+    'action': 'append',
+    'nargs': 2,
+    'help_text': (
+        'Tags to apply to objects uploaded to S3 by this operation, in '
+        'the format ``--tags key value``. Specify this parameter multiple '
+        'times, once for each tag. The tags are applied as the objects are '
+        'created, so no separate ``put-object-tagging`` call is made. They '
+        'are applied to every object which is part of this request. In a '
+        "sync, this means that files which haven't changed won't receive "
+        'the tags. This only affects objects uploaded to S3; it has no '
+        'effect on downloads, and copies between two S3 locations are '
+        'governed by ``--copy-props`` instead.'
+    )
+}
+
+
 METADATA_DIRECTIVE = {
     'name': 'metadata-directive', 'choices': ['COPY', 'REPLACE'],
     'help_text': (
@@ -528,9 +547,10 @@
                  SSE_C_COPY_SOURCE_KEY, STORAGE_CLASS, GRANTS,
                  WEBSITE_REDIRECT, CONTENT_TYPE, CACHE_CONTROL,
                  CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE,
-                 EXPIRES, SOURCE_REGION, ONLY_SHOW_ERRORS, NO_PROGRESS,
-                 PAGE_SIZE, IGNORE_GLACIER_WARNINGS, FORCE_GLACIER_TRANSFER,
-                 REQUEST_PAYER, CHECKSUM_MODE, CHECKSUM_ALGORITHM]
+                 EXPIRES, OBJECT_TAGS, SOURCE_REGION, ONLY_SHOW_ERRORS,
+                 NO_PROGRESS, PAGE_SIZE, IGNORE_GLACIER_WARNINGS,
+                 FORCE_GLACIER_TRANSFER, REQUEST_PAYER, CHECKSUM_MODE,
+                 CHECKSUM_ALGORITHM]
 
 
 def get_client(session, region, endpoint_url, verify, config=None):

awscli/customizations/s3/utils.py

@@ -19,6 +19,7 @@
 import re
 import time
 from collections import namedtuple, deque
+from urllib.parse import urlencode
 
 from dateutil.parser import parse
 from dateutil.tz import tzlocal, tzutc
@@ -475,6 +476,7 @@ def map_put_object_params(cls, request_params, cli_params):
         """Map CLI params to PutObject request params"""
         cls._set_general_object_params(request_params, cli_params)
         cls._set_metadata_params(request_params, cli_params)
+        cls._set_tagging_params(request_params, cli_params)
         cls._set_sse_request_params(request_params, cli_params)
         cls._set_sse_c_request_params(request_params, cli_params)
         cls._set_request_payer_param(request_params, cli_params)
@@ -526,6 +528,7 @@ def map_create_multipart_upload_params(cls, request_params, cli_params):
         cls._set_sse_request_params(request_params, cli_params)
         cls._set_sse_c_request_params(request_params, cli_params)
         cls._set_metadata_params(request_params, cli_params)
+        cls._set_tagging_params(request_params, cli_params)
         cls._set_request_payer_param(request_params, cli_params)
 
     @classmethod
@@ -615,6 +618,14 @@ def _set_metadata_params(cls, request_params, cli_params):
         if cli_params.get('metadata'):
             request_params['Metadata'] = cli_params['metadata']
 
+    @classmethod
+    def _set_tagging_params(cls, request_params, cli_params):
+        # Tagging is a URL-encoded query string built from the --tags pairs.
+        if cli_params.get('tags'):
+            request_params['Tagging'] = urlencode(
+                [(key, value) for key, value in cli_params['tags']]
+            )
+
     @classmethod
     def _auto_populate_metadata_directive(cls, request_params):
         if request_params.get('Metadata') and \

awscli/examples/s3/cp.rst

@@ -200,3 +200,12 @@ The following ``cp`` command downloads a single object (``mykey``) from the acce
 Output::
 
     download: s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/mykey to mydoc.txt
+
+**Example 16: Uploading an object and applying tags**
+
+The following ``cp`` command uploads a single file to a specified bucket and key and applies two tags to the object as
+it is created, so no separate ``put-object-tagging`` call is needed::
+
+    aws s3 cp test.txt s3://amzn-s3-demo-bucket/test.txt \
+        --tags Key1 Value1 \
+        --tags Key2 Value2

awscli/examples/s3/mv.rst

@@ -112,4 +112,17 @@ The following ``mv`` command moves a single file named ``mydoc.txt`` to the acce
 
 Output::
 
-    move: mydoc.txt to s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/mykey
+    move: mydoc.txt to s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/mykey
+
+**Example 11: Move a file to S3 and apply tags**
+
+The following ``mv`` command moves a single file to a specified bucket and key and applies two tags to the object as it
+is uploaded::
+
+    aws s3 mv file.txt s3://amzn-s3-demo-bucket/file.txt \
+        --tags Key1 Value1 \
+        --tags Key2 Value2
+
+Output::
+
+    move: file.txt to s3://amzn-s3-demo-bucket/file.txt

awscli/examples/s3/sync.rst

@@ -117,3 +117,16 @@ Output::
 
     upload: test.txt to s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/test.txt
     upload: test2.txt to s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/test2.txt
+
+**Example 9: Sync to S3 and apply tags**
+
+The following ``sync`` command syncs the current directory to a specified bucket and applies two tags to each object as
+it is uploaded. Because tags are applied only as objects are uploaded, files that are already in sync are not retagged::
+
+    aws s3 sync . s3://amzn-s3-demo-bucket \
+        --tags Key1 Value1 \
+        --tags Key2 Value2
+
+Output::
+
+    upload: test.txt to s3://amzn-s3-demo-bucket/test.txt

tests/functional/s3/test_cp_command.py

@@ -121,6 +121,43 @@ def test_upload_standard_ia(self):
         self.assertEqual(args['Bucket'], 'bucket')
         self.assertEqual(args['StorageClass'], 'STANDARD_IA')
 
+    def test_upload_with_tagging(self):
+        full_path = self.files.create_file('foo.txt', 'mycontent')
+        cmdline = (
+            '%s %s s3://bucket/key.txt --tags key1 value1 --tags key2 value2' %
+            (self.prefix, full_path))
+        self.parsed_responses = \
+            [{'ETag': '"c8afdb36c52cf4727836669019e69222"'}]
+        self.run_cmd(cmdline, expected_rc=0)
+        self.assertEqual(len(self.operations_called), 1,
+                         self.operations_called)
+        self.assertEqual(self.operations_called[0][0].name, 'PutObject')
+        self.assertEqual(self.operations_called[0][1]['Tagging'],
+                         'key1=value1&key2=value2')
+
+    def test_multipart_upload_with_tagging(self):
+        # The tag set must be applied once, on CreateMultipartUpload, and must
+        # not leak onto the individual UploadPart requests (S3 rejects it
+        # there).
+        full_path = self.files.create_file('foo.txt', 'a' * 10 * (1024 ** 2))
+        self.parsed_responses = [
+            {'UploadId': 'foo'},
+            {'ETag': '"foo-1"'},
+            {'ETag': '"foo-2"'},
+            {}
+        ]
+        cmdline = ('%s %s s3://bucket/key.txt'
+                   ' --tags key1 value1' % (self.prefix, full_path))
+        self.run_cmd(cmdline, expected_rc=0)
+        operations = [op[0].name for op in self.operations_called]
+        self.assertEqual(
+            operations,
+            ['CreateMultipartUpload', 'UploadPart', 'UploadPart',
+             'CompleteMultipartUpload'])
+        self.assertEqual(self.operations_called[0][1]['Tagging'],
+                         'key1=value1')
+        self.assertNotIn('Tagging', self.operations_called[1][1])
+
     def test_upload_onezone_ia(self):
         full_path = self.files.create_file('foo.txt', 'mycontent')
         cmdline = ('%s %s s3://bucket/key.txt --storage-class ONEZONE_IA' %

tests/functional/s3/test_mv_command.py

@@ -48,6 +48,19 @@ def test_website_redirect_ignore_paramfile(self):
             'http://someserver'
         )
 
+    def test_upload_with_tagging(self):
+        full_path = self.files.create_file('foo.txt', 'mycontent')
+        cmdline = (
+            '%s %s s3://bucket/key.txt --tags key1 value1 --tags key2 value2' %
+            (self.prefix, full_path))
+        self.parsed_responses = [{'ETag': '"c8afdb36c52cf4727836669019e69222"'}]
+        self.run_cmd(cmdline, expected_rc=0)
+        self.assertEqual(self.operations_called[0][0].name, 'PutObject')
+        self.assertEqual(
+            self.operations_called[0][1]['Tagging'],
+            'key1=value1&key2=value2'
+        )
+
     def test_metadata_directive_copy(self):
         self.parsed_responses = [
             {

tests/functional/s3/test_sync_command.py

@@ -43,6 +43,22 @@ def test_website_redirect_ignore_paramfile(self):
             'http://someserver'
         )
 
+    def test_upload_with_tagging(self):
+        self.files.create_file('foo.txt', 'mycontent')
+        cmdline = (
+            '%s %s s3://bucket/key.txt --tags key1 value1 --tags key2 value2' %
+            (self.prefix, self.files.rootdir))
+        self.parsed_responses = [
+            {"CommonPrefixes": [], "Contents": []},
+            {'ETag': '"c8afdb36c52cf4727836669019e69222"'}
+        ]
+        self.run_cmd(cmdline, expected_rc=0)
+        self.assertEqual(self.operations_called[1][0].name, 'PutObject')
+        self.assertEqual(
+            self.operations_called[1][1]['Tagging'],
+            'key1=value1&key2=value2'
+        )
+
     def test_no_recursive_option(self):
         cmdline = '. s3://mybucket --recursive'
         # Return code will be 2 for invalid parameter ``--recursive``

tests/unit/customizations/s3/test_utils.py

@@ -704,6 +704,33 @@ def test_upload_part_copy(self):
              'SSECustomerKey': 'my-sse-c-key'})
 
 
+class TestRequestParamsMapperTagging(unittest.TestCase):
+    def setUp(self):
+        self.cli_params = {'tags': [['key1', 'value1'], ['key2', 'value2']]}
+
+    def test_put_object(self):
+        params = {}
+        RequestParamsMapper.map_put_object_params(params, self.cli_params)
+        self.assertEqual(params, {'Tagging': 'key1=value1&key2=value2'})
+
+    def test_create_multipart_upload(self):
+        params = {}
+        RequestParamsMapper.map_create_multipart_upload_params(
+            params, self.cli_params)
+        self.assertEqual(params, {'Tagging': 'key1=value1&key2=value2'})
+
+    def test_tag_values_are_url_encoded(self):
+        params = {}
+        cli_params = {'tags': [['key', 'a value/with=specials']]}
+        RequestParamsMapper.map_put_object_params(params, cli_params)
+        self.assertEqual(params, {'Tagging': 'key=a+value%2Fwith%3Dspecials'})
+
+    def test_no_tags_param_is_a_noop(self):
+        params = {}
+        RequestParamsMapper.map_put_object_params(params, {})
+        self.assertEqual(params, {})
+
+
 class TestRequestParamsMapperChecksumAlgorithm:
     @pytest.fixture
     def cli_params(self):