HDDS-10435. Support S3 object tags for existing requests (apache#6607)

Author: ivandika3

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java

@@ -40,6 +40,8 @@ public enum OzoneManagerVersion implements ComponentVersion {
   LIGHTWEIGHT_LIST_KEYS(4, "OzoneManager version that supports lightweight"
       + " listKeys API."),
 
+  OBJECT_TAG(5, "OzoneManager version that supports object tags"),
+
   FUTURE_VERSION(-1, "Used internally in the client when the server side is "
       + " newer and an unknown server version has arrived to the client.");
 

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java

@@ -54,7 +54,6 @@
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -431,7 +430,7 @@ public void setEncryptionKey(String bekName) throws IOException {
   public OzoneOutputStream createKey(String key, long size)
       throws IOException {
     return createKey(key, size, defaultReplication,
-        new HashMap<>());
+        Collections.emptyMap());
   }
 
   /**
@@ -459,15 +458,35 @@ public OzoneOutputStream createKey(String key, long size,
    * @param key               Name of the key to be created.
    * @param size              Size of the data the key will point to.
    * @param replicationConfig Replication configuration.
+   * @param keyMetadata       Custom key metadata.
    * @return OzoneOutputStream to which the data has to be written.
    * @throws IOException
    */
   public OzoneOutputStream createKey(String key, long size,
       ReplicationConfig replicationConfig,
       Map<String, String> keyMetadata)
       throws IOException {
+    return this.createKey(key, size, replicationConfig, keyMetadata, Collections.emptyMap());
+  }
+
+  /**
+   * Creates a new key in the bucket.
+   *
+   * @param key               Name of the key to be created.
+   * @param size              Size of the data the key will point to.
+   * @param replicationConfig Replication configuration.
+   * @param keyMetadata       Custom key metadata.
+   * @param tags              Tags used for S3 object tags
+   * @return OzoneOutputStream to which the data has to be written.
+   * @throws IOException
+   */
+  public OzoneOutputStream createKey(String key, long size,
+      ReplicationConfig replicationConfig,
+      Map<String, String> keyMetadata,
+      Map<String, String> tags)
+      throws IOException {
     return proxy
-        .createKey(volumeName, name, key, size, replicationConfig, keyMetadata);
+        .createKey(volumeName, name, key, size, replicationConfig, keyMetadata, tags);
   }
 
   /**
@@ -491,6 +510,7 @@ public OzoneDataStreamOutput createStreamKey(String key, long size)
    * @param key               Name of the key to be created.
    * @param size              Size of the data the key will point to.
    * @param replicationConfig Replication configuration.
+   * @param keyMetadata       Custom key metadata.
    * @return OzoneDataStreamOutput to which the data has to be written.
    * @throws IOException
    */
@@ -500,8 +520,28 @@ public OzoneDataStreamOutput createStreamKey(String key, long size,
     if (replicationConfig == null) {
       replicationConfig = defaultReplication;
     }
+    return this.createStreamKey(key, size, replicationConfig, keyMetadata,
+        Collections.emptyMap());
+  }
+
+  /**
+   * Creates a new key in the bucket.
+   *
+   * @param key               Name of the key to be created.
+   * @param size              Size of the data the key will point to.
+   * @param replicationConfig Replication configuration.
+   * @param keyMetadata       Custom key metadata.
+   * @return OzoneDataStreamOutput to which the data has to be written.
+   * @throws IOException
+   */
+  public OzoneDataStreamOutput createStreamKey(String key, long size,
+      ReplicationConfig replicationConfig, Map<String, String> keyMetadata,
+      Map<String, String> tags) throws IOException {
+    if (replicationConfig == null) {
+      replicationConfig = defaultReplication;
+    }
     return proxy.createStreamKey(volumeName, name, key, size,
-        replicationConfig, keyMetadata);
+        replicationConfig, keyMetadata, tags);
   }
 
   /**
@@ -659,11 +699,12 @@ public void renameKeys(Map<String, String> keyMap)
 
   /**
    * Initiate multipart upload for a specified key.
-   * @param keyName
-   * @param type
-   * @param factor
+   * @param keyName Name of the key to be created when the multipart upload is completed.
+   * @param type Replication type to be used.
+   * @param factor Replication factor of the key.
    * @return OmMultipartInfo
    * @throws IOException
+   * @deprecated Use {@link OzoneBucket#initiateMultipartUpload(String, ReplicationConfig)} instead.
    */
   @Deprecated
   public OmMultipartInfo initiateMultipartUpload(String keyName,
@@ -676,6 +717,10 @@ public OmMultipartInfo initiateMultipartUpload(String keyName,
 
   /**
    * Initiate multipart upload for a specified key.
+   * @param keyName Name of the key to be created when the multipart upload is completed.
+   * @param config Replication config.
+   * @return OmMultipartInfo
+   * @throws IOException
    */
   public OmMultipartInfo initiateMultipartUpload(String keyName,
       ReplicationConfig config)
@@ -685,11 +730,32 @@ public OmMultipartInfo initiateMultipartUpload(String keyName,
 
   /**
    * Initiate multipart upload for a specified key.
+   * @param keyName Name of the key to be created when the multipart upload is completed.
+   * @param config Replication config.
+   * @param metadata Custom key metadata.
+   * @return OmMultipartInfo
+   * @throws IOException
    */
   public OmMultipartInfo initiateMultipartUpload(String keyName,
       ReplicationConfig config, Map<String, String> metadata)
       throws IOException {
-    return proxy.initiateMultipartUpload(volumeName, name, keyName, config, metadata);
+    return initiateMultipartUpload(keyName, config, metadata, Collections.emptyMap());
+  }
+
+  /**
+   * Initiate multipart upload for a specified key.
+   * @param keyName Name of the key to be created when the multipart upload is completed.
+   * @param config Replication config.
+   * @param metadata Custom key metadata.
+   * @param tags Tags used for S3 object tags.
+   * @return OmMultipartInfo
+   * @throws IOException
+   */
+  public OmMultipartInfo initiateMultipartUpload(String keyName,
+      ReplicationConfig config, Map<String, String> metadata,
+      Map<String, String> tags)
+      throws IOException {
+    return proxy.initiateMultipartUpload(volumeName, name, keyName, config, metadata, tags);
   }
 
   /**
@@ -1311,7 +1377,8 @@ private static OzoneKey toOzoneKey(OzoneFileStatusLight status) {
         keyInfo.getReplicationConfig(),
         metadata,
         keyInfo.isFile(),
-        keyInfo.getOwnerName());
+        keyInfo.getOwnerName(),
+        Collections.emptyMap());
   }
 
 

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java

@@ -63,7 +63,9 @@ public class OzoneKey {
 
   private ReplicationConfig replicationConfig;
 
-  private Map<String, String> metadata = new HashMap<>();
+  private final Map<String, String> metadata = new HashMap<>();
+
+  private final Map<String, String> tags = new HashMap<>();
 
   /**
    * Indicator if key is a file.
@@ -94,10 +96,12 @@ public OzoneKey(String volumeName, String bucketName,
   public OzoneKey(String volumeName, String bucketName,
                   String keyName, long size, long creationTime,
                   long modificationTime, ReplicationConfig replicationConfig,
-                  Map<String, String> metadata, boolean isFile, String owner) {
+                  Map<String, String> metadata, boolean isFile, String owner,
+                  Map<String, String> tags) {
     this(volumeName, bucketName, keyName, size, creationTime,
         modificationTime, replicationConfig, isFile, owner);
     this.metadata.putAll(metadata);
+    this.tags.putAll(tags);
   }
 
   /**
@@ -163,10 +167,24 @@ public Instant getModificationTime() {
     return modificationTime;
   }
 
+  /**
+   * Returns the metadata of the key.
+   *
+   * @return key metadata.
+   */
   public Map<String, String> getMetadata() {
     return metadata;
   }
 
+  /**
+   * Returns the tags of the key.
+   *
+   * @return key tags.
+   */
+  public Map<String, String> getTags() {
+    return tags;
+  }
+
   public void setMetadata(Map<String, String> metadata) {
     this.metadata.putAll(metadata);
   }
@@ -205,7 +223,8 @@ public static OzoneKey fromKeyInfo(OmKeyInfo keyInfo) {
     return new OzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(),
         keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(),
         keyInfo.getModificationTime(), keyInfo.getReplicationConfig(),
-        keyInfo.getMetadata(), keyInfo.isFile(), keyInfo.getOwnerName());
+        keyInfo.getMetadata(), keyInfo.isFile(), keyInfo.getOwnerName(),
+        keyInfo.getTags());
   }
 
 }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java

@@ -53,9 +53,9 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName,
       Map<String, String> metadata,
       FileEncryptionInfo feInfo,
       CheckedSupplier<OzoneInputStream, IOException> contentSupplier,
-      boolean isFile, String owner) {
+      boolean isFile, String owner, Map<String, String> tags) {
     super(volumeName, bucketName, keyName, size, creationTime,
-        modificationTime, replicationConfig, metadata, isFile, owner);
+        modificationTime, replicationConfig, metadata, isFile, owner, tags);
     this.ozoneKeyLocations = ozoneKeyLocations;
     this.feInfo = feInfo;
     this.contentSupplier = contentSupplier;

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -329,7 +329,7 @@ List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
    * @param size Size of the data
    * @param metadata Custom key value metadata
    * @return {@link OzoneOutputStream}
-   *
+   * @deprecated Use {@link ClientProtocol#createKey(String, String, String, long, ReplicationConfig, Map)} instead.
    */
   @Deprecated
   OzoneOutputStream createKey(String volumeName, String bucketName,
@@ -344,7 +344,7 @@ OzoneOutputStream createKey(String volumeName, String bucketName,
    * @param bucketName Name of the Bucket
    * @param keyName Name of the Key
    * @param size Size of the data
-   * @param metadata custom key value metadata
+   * @param metadata Custom key value metadata
    * @return {@link OzoneOutputStream}
    *
    */
@@ -353,6 +353,22 @@ OzoneOutputStream createKey(String volumeName, String bucketName,
       Map<String, String> metadata)
       throws IOException;
 
+  /**
+   * Writes a key in an existing bucket.
+   * @param volumeName Name of the Volume
+   * @param bucketName Name of the Bucket
+   * @param keyName Name of the Key
+   * @param size Size of the data
+   * @param metadata Custom key value metadata
+   * @param tags Tags used for S3 object tags
+   * @return {@link OzoneOutputStream}
+   *
+   */
+  OzoneOutputStream createKey(String volumeName, String bucketName,
+      String keyName, long size, ReplicationConfig replicationConfig,
+      Map<String, String> metadata, Map<String, String> tags)
+      throws IOException;
+
   /**
    * Writes a key in an existing bucket.
    * @param volumeName Name of the Volume
@@ -368,6 +384,22 @@ OzoneDataStreamOutput createStreamKey(String volumeName, String bucketName,
       Map<String, String> metadata)
       throws IOException;
 
+  /**
+   * Writes a key in an existing bucket.
+   * @param volumeName Name of the Volume
+   * @param bucketName Name of the Bucket
+   * @param keyName Name of the Key
+   * @param size Size of the data
+   * @param metadata custom key value metadata
+   * @param tags Tags used for S3 object tags
+   * @return {@link OzoneDataStreamOutput}
+   *
+   */
+  OzoneDataStreamOutput createStreamKey(String volumeName, String bucketName,
+      String keyName, long size, ReplicationConfig replicationConfig,
+      Map<String, String> metadata, Map<String, String> tags)
+      throws IOException;
+
   /**
    * Reads a key from an existing bucket.
    * @param volumeName Name of the Volume
@@ -535,6 +567,22 @@ OmMultipartInfo initiateMultipartUpload(String volumeName, String
       Map<String, String> metadata)
       throws IOException;
 
+  /**
+   * Initiate Multipart upload.
+   * @param volumeName Name of the Volume
+   * @param bucketName Name of the Bucket
+   * @param keyName Name of the Key
+   * @param replicationConfig Replication config
+   * @param metadata Custom key value metadata
+   * @param tags Tags used for S3 object tags
+   * @return {@link OmMultipartInfo}
+   * @throws IOException
+   */
+  OmMultipartInfo initiateMultipartUpload(String volumeName, String
+      bucketName, String keyName, ReplicationConfig replicationConfig,
+      Map<String, String> metadata, Map<String, String> tags)
+      throws IOException;
+
   /**
    * Create a part key for a multipart upload key.
    * @param volumeName