From 48dcfaeb457b46373022d2870e7bd1cd42e1ced4 Mon Sep 17 00:00:00 2001 From: Colm Dougan Date: Wed, 14 Jan 2026 17:25:58 +0000 Subject: [PATCH] HDDS-14004. EventNotification: Capture data to the completed operation ledger table --- .../om/ratis/OzoneManagerDoubleBuffer.java | 42 +++++++ .../om/request/file/OMFileCreateRequest.java | 3 +- .../file/OMFileCreateRequestWithFSO.java | 3 +- .../key/OMKeyRenameRequestWithFSO.java | 3 +- .../om/response/HasCompletedRequestInfo.java | 29 +++++ .../bucket/OMBucketCreateResponse.java | 16 ++- .../bucket/OMBucketDeleteResponse.java | 17 ++- .../file/OMDirectoryCreateResponse.java | 18 ++- .../response/file/OMFileCreateResponse.java | 20 ++- .../file/OMFileCreateResponseWithFSO.java | 16 ++- .../om/response/key/OMKeyCommitResponse.java | 18 ++- .../om/response/key/OMKeyCreateResponse.java | 26 +++- .../key/OMKeyCreateResponseWithFSO.java | 12 ++ .../om/response/key/OMKeyDeleteResponse.java | 17 ++- .../om/response/key/OMKeyRenameResponse.java | 28 ++++- .../key/OMKeyRenameResponseWithFSO.java | 17 ++- .../volume/OMVolumeCreateResponse.java | 16 ++- .../volume/OMVolumeDeleteResponse.java | 17 ++- ...zoneManagerDoubleBufferWithOMResponse.java | 115 ++++++++++++++++++ .../snapshot/TestOMSnapshotCreateRequest.java | 2 +- .../bucket/TestOMBucketCreateResponse.java | 28 +++++ .../bucket/TestOMBucketDeleteResponse.java | 28 +++++ .../file/TestOMDirectoryCreateResponse.java | 36 ++++++ .../file/TestOMFileCreateResponseWithFSO.java | 12 ++ .../response/key/TestOMKeyCommitResponse.java | 29 +++++ .../response/key/TestOMKeyCreateResponse.java | 34 ++++++ .../key/TestOMKeyCreateResponseWithFSO.java | 6 + .../response/key/TestOMKeyDeleteResponse.java | 30 +++++ .../response/key/TestOMKeyRenameResponse.java | 35 ++++++ .../key/TestOMKeyRenameResponseWithFSO.java | 3 +- 30 files changed, 656 insertions(+), 20 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/HasCompletedRequestInfo.java diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java index 1f77f6f5b495..7e1257740cf3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java @@ -44,7 +44,9 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.S3SecretManager; import org.apache.hadoop.ozone.om.codec.OMDBDefinition; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.HasCompletedRequestInfo; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -409,6 +411,46 @@ private String addToBatch(Queue buffer, BatchOperation batchOperation) { try { addToBatchWithTrace(omResponse, () -> response.checkAndUpdateDB(omMetadataManager, batchOperation)); + + // This is a strawman approach and requires some discussion + // with the community on approach. + // + // At the moment any response type which we want to capture a + // OmCompletedRequestInfo for is required to implement the + // interface HasCompletedRequestInfo and the method + // getCompletedRequestInfo() and we then have this extra step + // here in the double buffer to capture the rows. + // + // It would seem ideal that the double buffer shouldn't have to + // know/care that there is the concept of capturing this + // OmCompletedRequestInfo row for certain response times but the + // approach described above seemed like the least invasive + // approach overall. I am open to other views. + // + // Other approaches I considered: + // - adding a similar getCompletedRequestInfo method to each + // *request* type which want to capture a row for. The downside + // of this was that since requests are not part of the + // addToBatch flow the OmCompletedRequestInfo instance then had + // to be passed through from the request to the relevant + // response constructors and this created quite a bit of code + // churn which I perceived as messy + // + // * in terms of the actual data capture, rather than this + // "instanceof" approach in this class I toyed with + // having each response type which we want to capture a row for + // capturing it itself in its on implementation of addDBBatch + // (i.e. no need for any new code in this class). This + // seemed to be a bit messier to me in terms of code duplication + // + if (response instanceof HasCompletedRequestInfo) { + OmCompletedRequestInfo completedRequestInfo = ((HasCompletedRequestInfo) response).getCompletedRequestInfo( + entry.getTermIndex().getIndex()); + + omMetadataManager.getCompletedRequestInfoTable().putWithBatch( + batchOperation, completedRequestInfo.getTrxLogIndex(), completedRequestInfo); + } + } catch (IOException ex) { // During Adding to RocksDB batch entry got an exception. // We should terminate the OM. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java index 9788cfbafe17..9f89e327d8a2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java @@ -295,7 +295,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut .setOpenVersion(openVersion).build()) .setCmdType(CreateFile); omClientResponse = new OMFileCreateResponse(omResponse.build(), - omKeyInfo, missingParentInfos, clientID, omBucketInfo.copyObject()); + omKeyInfo, missingParentInfos, clientID, omBucketInfo.copyObject(), + isRecursive, isOverWrite); result = Result.SUCCESS; } catch (IOException | InvalidPathException ex) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestWithFSO.java index 6036fe90dbb7..e4b662e09b6b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestWithFSO.java @@ -216,7 +216,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut .setCmdType(Type.CreateFile); omClientResponse = new OMFileCreateResponseWithFSO(omResponse.build(), omFileInfo, missingParentInfos, clientID, - omBucketInfo.copyObject(), volumeId); + omBucketInfo.copyObject(), volumeId, + isRecursive, isOverWrite); result = Result.SUCCESS; } catch (IOException | InvalidPathException ex) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java index 4e93f8d1ba59..05a4e5a60785 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java @@ -342,7 +342,8 @@ private OMClientResponse renameKey(OmKeyInfo toKeyParent, String toKeyName, OMClientResponse omClientResponse = new OMKeyRenameResponseWithFSO( omResponse.setRenameKeyResponse(RenameKeyResponse.newBuilder()).build(), dbFromKey, dbToKey, fromKeyParent, toKeyParent, fromKeyValue, - omBucketInfo, isRenameDirectory, getBucketLayout()); + omBucketInfo, isRenameDirectory, getBucketLayout(), + fromKeyName, toKeyName); return omClientResponse; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/HasCompletedRequestInfo.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/HasCompletedRequestInfo.java new file mode 100644 index 000000000000..f3d415e1b0ee --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/HasCompletedRequestInfo.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response; + +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; + +/** + * Interface to define that a response class provides a + * OmCompletedRequestInfo implementation. + */ +public interface HasCompletedRequestInfo { + + OmCompletedRequestInfo getCompletedRequestInfo(long trxnLogIndex); +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/bucket/OMBucketCreateResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/bucket/OMBucketCreateResponse.java index 2bb5034d0706..df33307e921d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/bucket/OMBucketCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/bucket/OMBucketCreateResponse.java @@ -26,16 +26,19 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.HasCompletedRequestInfo; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; /** * Response for CreateBucket request. */ @CleanupTableInfo(cleanupTables = {BUCKET_TABLE, VOLUME_TABLE}) -public final class OMBucketCreateResponse extends OMClientResponse { +public final class OMBucketCreateResponse extends OMClientResponse implements HasCompletedRequestInfo { private final OmBucketInfo omBucketInfo; private final OmVolumeArgs omVolumeArgs; @@ -87,5 +90,16 @@ public OmBucketInfo getOmBucketInfo() { return omBucketInfo; } + @Override + public OmCompletedRequestInfo getCompletedRequestInfo(long trxnLogIndex) { + return OmCompletedRequestInfo.newBuilder() + .setTrxLogIndex(trxnLogIndex) + .setCmdType(Type.CreateBucket) + .setCreationTime(System.currentTimeMillis()) + .setVolumeName(omBucketInfo.getVolumeName()) + .setBucketName(omBucketInfo.getBucketName()) + .setOpArgs(new OmCompletedRequestInfo.OperationArgs.NoArgs()) + .build(); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/bucket/OMBucketDeleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/bucket/OMBucketDeleteResponse.java index e72ebf75a32c..097b6079798b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/bucket/OMBucketDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/bucket/OMBucketDeleteResponse.java @@ -24,16 +24,19 @@ import java.io.IOException; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.HasCompletedRequestInfo; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; /** * Response for DeleteBucket request. */ @CleanupTableInfo(cleanupTables = {BUCKET_TABLE, VOLUME_TABLE}) -public final class OMBucketDeleteResponse extends OMClientResponse { +public final class OMBucketDeleteResponse extends OMClientResponse implements HasCompletedRequestInfo { private String volumeName; private String bucketName; @@ -89,5 +92,15 @@ public String getBucketName() { return bucketName; } + @Override + public OmCompletedRequestInfo getCompletedRequestInfo(long trxnLogIndex) { + return OmCompletedRequestInfo.newBuilder() + .setTrxLogIndex(trxnLogIndex) + .setCmdType(Type.DeleteBucket) + .setCreationTime(System.currentTimeMillis()) + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setOpArgs(new OmCompletedRequestInfo.OperationArgs.NoArgs()) + .build(); + } } - diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMDirectoryCreateResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMDirectoryCreateResponse.java index f6e3a9d5de2f..858a3cb680da 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMDirectoryCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMDirectoryCreateResponse.java @@ -26,11 +26,14 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.file.OMDirectoryCreateRequest.Result; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.HasCompletedRequestInfo; import org.apache.hadoop.ozone.om.response.key.OmKeyResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +41,7 @@ * Response for create directory request. */ @CleanupTableInfo(cleanupTables = {KEY_TABLE}) -public class OMDirectoryCreateResponse extends OmKeyResponse { +public class OMDirectoryCreateResponse extends OmKeyResponse implements HasCompletedRequestInfo { private static final Logger LOG = LoggerFactory.getLogger(OMDirectoryCreateResponse.class); @@ -98,4 +101,17 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, LOG.debug("Directory already exists. addToDBBatch is a no-op"); } } + + @Override + public OmCompletedRequestInfo getCompletedRequestInfo(long trxnLogIndex) { + return OmCompletedRequestInfo.newBuilder() + .setTrxLogIndex(trxnLogIndex) + .setCmdType(Type.CreateDirectory) + .setCreationTime(System.currentTimeMillis()) + .setVolumeName(dirKeyInfo.getVolumeName()) + .setBucketName(dirKeyInfo.getBucketName()) + .setKeyName(dirKeyInfo.getKeyName()) + .setOpArgs(new OmCompletedRequestInfo.OperationArgs.NoArgs()) + .build(); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponse.java index 428e90075b36..55b840e91e44 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponse.java @@ -24,10 +24,12 @@ import java.util.List; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; /** * Response for crate file request. @@ -35,12 +37,19 @@ @CleanupTableInfo(cleanupTables = {KEY_TABLE, OPEN_KEY_TABLE}) public class OMFileCreateResponse extends OMKeyCreateResponse { + private boolean isRecursive; + private boolean isOverWrite; + public OMFileCreateResponse(@Nonnull OMResponse omResponse, @Nonnull OmKeyInfo omKeyInfo, @Nonnull List parentKeyInfos, long openKeySessionID, - @Nonnull OmBucketInfo omBucketInfo) { + @Nonnull OmBucketInfo omBucketInfo, + boolean isRecursive, boolean isOverWrite) { super(omResponse, omKeyInfo, parentKeyInfos, openKeySessionID, omBucketInfo); + + this.isRecursive = isRecursive; + this.isOverWrite = isOverWrite; } /** @@ -53,4 +62,13 @@ public OMFileCreateResponse(@Nonnull OMResponse omResponse, @Nonnull checkStatusNotOK(); } + @Override + public Type getOperationType() { + return Type.CreateFile; + } + + @Override + public OmCompletedRequestInfo.OperationArgs getCompletedRequestInfoArgs() { + return new OmCompletedRequestInfo.OperationArgs.CreateFileArgs(isRecursive, isOverWrite); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseWithFSO.java index 02c72110ad4d..2ffadbe0553b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseWithFSO.java @@ -42,6 +42,9 @@ BUCKET_TABLE}) public class OMFileCreateResponseWithFSO extends OMFileCreateResponse { + private static final boolean DEFAULT_IS_RECURSIVE = false; + private static final boolean DEFAULT_IS_OVERWRITE = false; + private List parentDirInfos; private long volumeId; @@ -49,8 +52,19 @@ public OMFileCreateResponseWithFSO(@Nonnull OMResponse omResponse, @Nonnull OmKeyInfo omKeyInfo, @Nonnull List parentDirInfos, long openKeySessionID, @Nonnull OmBucketInfo omBucketInfo, @Nonnull long volumeId) { + this(omResponse, omKeyInfo, parentDirInfos, openKeySessionID, + omBucketInfo, volumeId, DEFAULT_IS_RECURSIVE, + DEFAULT_IS_OVERWRITE); + } + + @SuppressWarnings("checkstyle:ParameterNumber") + public OMFileCreateResponseWithFSO(@Nonnull OMResponse omResponse, + @Nonnull OmKeyInfo omKeyInfo, + @Nonnull List parentDirInfos, long openKeySessionID, + @Nonnull OmBucketInfo omBucketInfo, @Nonnull long volumeId, + boolean isRecursive, boolean isOverWrite) { super(omResponse, omKeyInfo, new ArrayList<>(), openKeySessionID, - omBucketInfo); + omBucketInfo, isRecursive, isOverWrite); this.parentDirInfos = parentDirInfos; this.volumeId = volumeId; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponse.java index 425c4f63ac5e..8df65bc87e02 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponse.java @@ -30,17 +30,20 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.HasCompletedRequestInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; /** * Response for CommitKey request. */ @CleanupTableInfo(cleanupTables = {OPEN_KEY_TABLE, KEY_TABLE, DELETED_TABLE, BUCKET_TABLE}) -public class OMKeyCommitResponse extends OmKeyResponse { +public class OMKeyCommitResponse extends OmKeyResponse implements HasCompletedRequestInfo { private OmKeyInfo omKeyInfo; private String ozoneKeyName; @@ -154,4 +157,17 @@ protected boolean isHSync() { public OmKeyInfo getNewOpenKeyInfo() { return newOpenKeyInfo; } + + @Override + public OmCompletedRequestInfo getCompletedRequestInfo(long trxnLogIndex) { + return OmCompletedRequestInfo.newBuilder() + .setTrxLogIndex(trxnLogIndex) + .setCmdType(Type.CommitKey) + .setCreationTime(System.currentTimeMillis()) + .setVolumeName(omKeyInfo.getVolumeName()) + .setBucketName(omKeyInfo.getBucketName()) + .setKeyName(omKeyInfo.getKeyName()) + .setOpArgs(new OmCompletedRequestInfo.OperationArgs.NoArgs()) + .build(); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponse.java index 1bd03c4187be..d5873f1efb9a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponse.java @@ -28,9 +28,12 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.HasCompletedRequestInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +41,7 @@ * Response for CreateKey request. */ @CleanupTableInfo(cleanupTables = {OPEN_KEY_TABLE, KEY_TABLE, BUCKET_TABLE}) -public class OMKeyCreateResponse extends OmKeyResponse { +public class OMKeyCreateResponse extends OmKeyResponse implements HasCompletedRequestInfo { protected static final Logger LOG = LoggerFactory.getLogger(OMKeyCreateResponse.class); @@ -114,5 +117,26 @@ protected OmKeyInfo getOmKeyInfo() { protected OmBucketInfo getOmBucketInfo() { return omBucketInfo; } + + protected Type getOperationType() { + return Type.CreateKey; + } + + protected OmCompletedRequestInfo.OperationArgs getCompletedRequestInfoArgs() { + return new OmCompletedRequestInfo.OperationArgs.NoArgs(); + } + + @Override + public OmCompletedRequestInfo getCompletedRequestInfo(long trxnLogIndex) { + return OmCompletedRequestInfo.newBuilder() + .setTrxLogIndex(trxnLogIndex) + .setCmdType(getOperationType()) + .setCreationTime(System.currentTimeMillis()) + .setVolumeName(omKeyInfo.getVolumeName()) + .setBucketName(omKeyInfo.getBucketName()) + .setKeyName(omKeyInfo.getKeyName()) + .setOpArgs(getCompletedRequestInfoArgs()) + .build(); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponseWithFSO.java index a72ad370b940..c3d3614e5542 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponseWithFSO.java @@ -25,11 +25,13 @@ import java.util.List; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponseWithFSO; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; /** * Response for CreateKey request - prefix layout. @@ -55,4 +57,14 @@ public OMKeyCreateResponseWithFSO(@Nonnull OMResponse omResponse, @Nonnull BucketLayout bucketLayout) { super(omResponse, bucketLayout); } + + @Override + public Type getOperationType() { + return Type.CreateKey; + } + + @Override + public OmCompletedRequestInfo.OperationArgs getCompletedRequestInfoArgs() { + return new OmCompletedRequestInfo.OperationArgs.NoArgs(); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java index be646c4ee289..b4dd59953feb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java @@ -30,15 +30,18 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.HasCompletedRequestInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; /** * Response for DeleteKey request. */ @CleanupTableInfo(cleanupTables = {KEY_TABLE, OPEN_KEY_TABLE, DELETED_TABLE, BUCKET_TABLE}) -public class OMKeyDeleteResponse extends AbstractOMKeyDeleteResponse { +public class OMKeyDeleteResponse extends AbstractOMKeyDeleteResponse implements HasCompletedRequestInfo { private OmKeyInfo omKeyInfo; private OmBucketInfo omBucketInfo; @@ -107,4 +110,16 @@ protected OmBucketInfo getOmBucketInfo() { return omBucketInfo; } + @Override + public OmCompletedRequestInfo getCompletedRequestInfo(long trxnLogIndex) { + return OmCompletedRequestInfo.newBuilder() + .setTrxLogIndex(trxnLogIndex) + .setCmdType(Type.DeleteKey) + .setCreationTime(System.currentTimeMillis()) + .setVolumeName(omKeyInfo.getVolumeName()) + .setBucketName(omKeyInfo.getBucketName()) + .setKeyName(omKeyInfo.getKeyName()) + .setOpArgs(new OmCompletedRequestInfo.OperationArgs.NoArgs()) + .build(); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java index 894a80328752..4b64952d3440 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java @@ -25,16 +25,19 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.OMClientRequestUtils; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.HasCompletedRequestInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; /** * Response for RenameKey request. */ @CleanupTableInfo(cleanupTables = {KEY_TABLE, SNAPSHOT_RENAMED_TABLE}) -public class OMKeyRenameResponse extends OmKeyResponse { +public class OMKeyRenameResponse extends OmKeyResponse implements HasCompletedRequestInfo { private String fromKeyName; private String toKeyName; @@ -107,4 +110,27 @@ public String getFromKeyName() { public String getToKeyName() { return toKeyName; } + + protected String getFromKeyNameInRequest() { + return fromKeyName; + } + + protected String getToKeyNameInRequest() { + return toKeyName; + } + + @Override + public OmCompletedRequestInfo getCompletedRequestInfo(long trxnLogIndex) { + String fromKeyNameInRequest = getFromKeyNameInRequest(); + String toKeyNameInRequest = getToKeyNameInRequest(); + return OmCompletedRequestInfo.newBuilder() + .setTrxLogIndex(trxnLogIndex) + .setCmdType(Type.RenameKey) + .setCreationTime(System.currentTimeMillis()) + .setVolumeName(renameKeyInfo.getVolumeName()) + .setBucketName(renameKeyInfo.getBucketName()) + .setKeyName(fromKeyNameInRequest) + .setOpArgs(new OmCompletedRequestInfo.OperationArgs.RenameKeyArgs(toKeyNameInRequest)) + .build(); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseWithFSO.java index e9f2a32776e5..05af71c421f5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseWithFSO.java @@ -45,18 +45,23 @@ public class OMKeyRenameResponseWithFSO extends OMKeyRenameResponse { private OmKeyInfo fromKeyParent; private OmKeyInfo toKeyParent; private OmBucketInfo bucketInfo; + private String fromKeyNameInRequest; + private String toKeyNameInRequest; @SuppressWarnings("checkstyle:ParameterNumber") public OMKeyRenameResponseWithFSO(@Nonnull OMResponse omResponse, String fromDBKey, String toDBKey, OmKeyInfo fromKeyParent, OmKeyInfo toKeyParent, @Nonnull OmKeyInfo renameKeyInfo, OmBucketInfo bucketInfo, - boolean isRenameDirectory, BucketLayout bucketLayout) { + boolean isRenameDirectory, BucketLayout bucketLayout, + String fromKeyNameInRequest, String toKeyNameInRequest) { super(omResponse, fromDBKey, toDBKey, renameKeyInfo, bucketLayout); this.isRenameDirectory = isRenameDirectory; this.fromKeyParent = fromKeyParent; this.toKeyParent = toKeyParent; this.bucketInfo = bucketInfo; + this.fromKeyNameInRequest = fromKeyNameInRequest; + this.toKeyNameInRequest = toKeyNameInRequest; } /** @@ -126,6 +131,16 @@ public BucketLayout getBucketLayout() { return BucketLayout.FILE_SYSTEM_OPTIMIZED; } + @Override + public String getFromKeyNameInRequest() { + return fromKeyNameInRequest; + } + + @Override + public String getToKeyNameInRequest() { + return toKeyNameInRequest; + } + private void addDirToDBBatch(OMMetadataManager metadataManager, OmKeyInfo keyInfo, long volumeId, long bucketId, BatchOperation batch) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/volume/OMVolumeCreateResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/volume/OMVolumeCreateResponse.java index 60b0029d777c..79902210e5d7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/volume/OMVolumeCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/volume/OMVolumeCreateResponse.java @@ -24,17 +24,20 @@ import java.io.IOException; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.HasCompletedRequestInfo; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.ozone.storage.proto.OzoneManagerStorageProtos.PersistedUserVolumeInfo; /** * Response for CreateVolume request. */ @CleanupTableInfo(cleanupTables = VOLUME_TABLE) -public class OMVolumeCreateResponse extends OMClientResponse { +public class OMVolumeCreateResponse extends OMClientResponse implements HasCompletedRequestInfo { private PersistedUserVolumeInfo userVolumeInfo; private OmVolumeArgs omVolumeArgs; @@ -76,5 +79,14 @@ public OmVolumeArgs getOmVolumeArgs() { return omVolumeArgs; } + @Override + public OmCompletedRequestInfo getCompletedRequestInfo(long trxnLogIndex) { + return OmCompletedRequestInfo.newBuilder() + .setTrxLogIndex(trxnLogIndex) + .setCmdType(Type.CreateVolume) + .setCreationTime(System.currentTimeMillis()) + .setVolumeName(omVolumeArgs.getVolume()) + .setOpArgs(new OmCompletedRequestInfo.OperationArgs.NoArgs()) + .build(); + } } - diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/volume/OMVolumeDeleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/volume/OMVolumeDeleteResponse.java index cb731dd9a33a..ecd72336aa5e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/volume/OMVolumeDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/volume/OMVolumeDeleteResponse.java @@ -23,16 +23,19 @@ import java.io.IOException; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.HasCompletedRequestInfo; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.ozone.storage.proto.OzoneManagerStorageProtos.PersistedUserVolumeInfo; /** * Response for DeleteVolume request. */ @CleanupTableInfo(cleanupTables = {VOLUME_TABLE}) -public class OMVolumeDeleteResponse extends OMClientResponse { +public class OMVolumeDeleteResponse extends OMClientResponse implements HasCompletedRequestInfo { private String volume; private String owner; private PersistedUserVolumeInfo updatedVolumeList; @@ -71,5 +74,15 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, omMetadataManager.getVolumeTable().deleteWithBatch(batchOperation, omMetadataManager.getVolumeKey(volume)); } -} + @Override + public OmCompletedRequestInfo getCompletedRequestInfo(long trxnLogIndex) { + return OmCompletedRequestInfo.newBuilder() + .setTrxLogIndex(trxnLogIndex) + .setCmdType(Type.DeleteVolume) + .setCreationTime(System.currentTimeMillis()) + .setVolumeName(volume) + .setOpArgs(new OmCompletedRequestInfo.OperationArgs.NoArgs()) + .build(); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java index 4ba3c6b5d1b7..2cb73e393199 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java @@ -33,10 +33,14 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import java.util.Queue; import java.util.UUID; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicLong; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StorageTypeProto; import org.apache.hadoop.hdds.utils.TransactionInfo; @@ -52,6 +56,7 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.request.bucket.OMBucketCreateRequest; @@ -63,6 +68,7 @@ import org.apache.hadoop.ozone.om.response.volume.OMVolumeCreateResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Daemon; import org.apache.ozone.test.GenericTestUtils; @@ -181,6 +187,9 @@ public void testDoubleBufferWithMixOfTransactions() throws Exception { assertEquals(5, omMetadataManager.countRowsInTable( omMetadataManager.getBucketTable())); + assertEquals(16, omMetadataManager.countRowsInTable( + omMetadataManager.getCompletedRequestInfoTable())); + // Now after this in our DB we should have 5 buckets and one volume checkVolume(volumeName, omVolumeCreateResponse); @@ -195,6 +204,61 @@ public void testDoubleBufferWithMixOfTransactions() throws Exception { 100, 30000); } + @Test + public void testDoubleBufferWithMixOfTransactionsCapturesCompletedInfoRows() throws Exception { + // This test checks count, data in table is correct or not. + Queue< OMBucketCreateResponse> bucketQueue = + new ConcurrentLinkedQueue<>(); + Queue< Pair > bucketWithTxnIdQueue = + new ConcurrentLinkedQueue<>(); + + String volumeName = UUID.randomUUID().toString(); + long createVolumeTransactionID = trxId.incrementAndGet(); + OMVolumeCreateResponse omVolumeCreateResponse = + (OMVolumeCreateResponse) createVolume(volumeName, + createVolumeTransactionID); + + // do some transactions + int bucketCount = 10; + for (int i = 0; i < 10; i++) { + String bucketName = UUID.randomUUID().toString(); + long transactionID = trxId.incrementAndGet(); + OMBucketCreateResponse omBucketCreateResponse = createBucket(volumeName, + bucketName, transactionID); + + bucketQueue.add(omBucketCreateResponse); + bucketWithTxnIdQueue.add(new ImmutablePair<>(transactionID, omBucketCreateResponse)); + } + + // We are doing +1 for volume transaction. + GenericTestUtils.waitFor( + () -> doubleBuffer.getFlushedTransactionCountForTesting() == bucketCount + 1, + 100, 120000); + + assertEquals(1, omMetadataManager.countRowsInTable( + omMetadataManager.getVolumeTable())); + + assertEquals(10, omMetadataManager.countRowsInTable( + omMetadataManager.getBucketTable())); + + assertEquals(11, omMetadataManager.countRowsInTable( + omMetadataManager.getCompletedRequestInfoTable())); + + // Now after this in our DB we should have one volume + 10 buckets + checkVolume(volumeName, omVolumeCreateResponse); + + checkCreateBuckets(bucketQueue); + + // ... and one volume + 10 buckets worth of OmCompletedRequestInfo + // objects captured + checkCompletedRequestInfos(createVolumeTransactionID, volumeName, bucketWithTxnIdQueue); + + // Check lastAppliedIndex is updated correctly or not. + final long expectedIndex = bucketCount + 1; + GenericTestUtils.waitFor(() -> assertTransactionInfo(expectedIndex), + 100, 30000); + } + private boolean assertTransactionInfo(long lastAppliedIndex) { final TransactionInfo info; try { @@ -259,6 +323,9 @@ public void testDoubleBufferWithMixOfTransactionsParallel() throws Exception { assertEquals(10, omMetadataManager.countRowsInTable( omMetadataManager.getBucketTable())); + assertEquals(32, omMetadataManager.countRowsInTable( + omMetadataManager.getCompletedRequestInfoTable())); + // Now after this in our DB we should have 5 buckets and one volume @@ -377,6 +444,54 @@ private void checkDeletedBuckets(Queue })); } + private void checkCompletedRequestInfos(long createVolumeTxnId, String volumeName, + Queue> bucketQueue) throws Exception { + + List ledgerEntries = collectCompletedRequestInfoTableEntries(); + assertThat(ledgerEntries).hasSize(1 + bucketQueue.size()); + + // The first entry should be the CreateVolume response + OmCompletedRequestInfo volumeLedgerEntry = ledgerEntries.get(0); + assertThat(volumeLedgerEntry.getTrxLogIndex()).isEqualTo(createVolumeTxnId); + assertThat(volumeLedgerEntry.getCmdType()).isEqualTo(Type.CreateVolume); + assertThat(volumeLedgerEntry.getVolumeName()).isEqualTo(volumeName); + assertThat(volumeLedgerEntry.getBucketName()).isEqualTo(""); + assertThat(volumeLedgerEntry.getKeyName()).isEqualTo(""); + assertThat(volumeLedgerEntry.getOpArgs()).isEqualTo( + new OmCompletedRequestInfo.OperationArgs.NoArgs()); + + // Check all subsequent entries (CreateBucket) against the queue of + // responses + List bucketEntries = ledgerEntries.subList(1, ledgerEntries.size()); + assertThat(bucketEntries).zipSatisfy(bucketQueue, (bucketLedgerEntry, createBucketPair) -> { + long createBucketTxnId = createBucketPair.getLeft(); + OmBucketInfo omBucketInfo = createBucketPair.getRight().getOmBucketInfo(); + + assertThat(bucketLedgerEntry.getTrxLogIndex()).isEqualTo(createBucketTxnId); + assertThat(bucketLedgerEntry.getCmdType()).isEqualTo(Type.CreateBucket); + assertThat(bucketLedgerEntry.getVolumeName()).isEqualTo(volumeName); + assertThat(bucketLedgerEntry.getBucketName()).isEqualTo(omBucketInfo.getBucketName()); + assertThat(bucketLedgerEntry.getKeyName()).isEqualTo(""); + assertThat(bucketLedgerEntry.getOpArgs()).isEqualTo( + new OmCompletedRequestInfo.OperationArgs.NoArgs()); + }); + } + + private List collectCompletedRequestInfoTableEntries() throws Exception { + List ledgerEntries = new ArrayList<>(); + + try (Table.KeyValueIterator + tableIter = omMetadataManager.getCompletedRequestInfoTable().iterator()) { + + while (tableIter.hasNext()) { + Table.KeyValue tableRow = tableIter.next(); + ledgerEntries.add(tableRow.getValue()); + } + } + + return ledgerEntries; + } + /** * Create bucketCount number of createBucket responses for each iteration. * All these iterations are run in parallel. Then verify OM DB has correct diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java index 4136f4da20f5..c06f82619e5b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java @@ -518,7 +518,7 @@ private void renameDirInBucket(String volumeName, String bucketName, String from OMKeyRenameResponseWithFSO omKeyRenameResponse = new OMKeyRenameResponseWithFSO(omResponse, getDBKeyName(fromKeyInfo), getDBKeyName(toKeyInfo), fromKeyParent, null, toKeyInfo, - null, true, BucketLayout.FILE_SYSTEM_OPTIMIZED); + null, true, BucketLayout.FILE_SYSTEM_OPTIMIZED, fromKey, toKey); omKeyRenameResponse.addToDBBatch(getOmMetadataManager(), getBatchOperation()); getOmMetadataManager().getStore().commitBatchOperation(getBatchOperation()); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketCreateResponse.java index 7ae5f878a268..726534ea2c0f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketCreateResponse.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om.response.bucket; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.file.Path; import java.util.UUID; @@ -28,6 +29,7 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.response.TestOMResponseUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateBucketResponse; @@ -95,4 +97,30 @@ public void testAddToDBBatch() throws Exception { bucketName), keyValue.getKey()); assertEquals(omBucketInfo, keyValue.getValue()); } + + @Test + void testGetCompletedRequestInfo() { + long txIndex = 100L; + + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + OmBucketInfo omBucketInfo = TestOMResponseUtils.createBucket( + volumeName, bucketName); + + OMBucketCreateResponse omBucketCreateResponse = + new OMBucketCreateResponse(OMResponse.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.CreateBucket) + .setStatus(OzoneManagerProtocolProtos.Status.OK) + .setCreateBucketResponse( + CreateBucketResponse.newBuilder().build()).build(), + omBucketInfo); + + OmCompletedRequestInfo info = omBucketCreateResponse.getCompletedRequestInfo(txIndex); + + assertEquals(txIndex, info.getTrxLogIndex()); + assertEquals(OzoneManagerProtocolProtos.Type.CreateBucket, info.getCmdType()); + assertEquals(volumeName, info.getVolumeName()); + assertEquals(bucketName, info.getBucketName()); + assertTrue(info.getOpArgs() instanceof OmCompletedRequestInfo.OperationArgs.NoArgs); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java index 3699b91cd275..deb5ac927f58 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java @@ -17,7 +17,9 @@ package org.apache.hadoop.ozone.om.response.bucket; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.file.Path; import java.util.UUID; @@ -27,6 +29,7 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.response.TestOMResponseUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateBucketResponse; @@ -96,4 +99,29 @@ public void testAddToDBBatch() throws Exception { omMetadataManager.getBucketKey(volumeName, bucketName))); } + @Test + void testGetCompletedRequestInfo() { + long txIndex = 100L; + + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + OmBucketInfo omBucketInfo = TestOMResponseUtils.createBucket( + volumeName, bucketName); + + OMBucketDeleteResponse omBucketDeleteResponse = + new OMBucketDeleteResponse(OMResponse.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.DeleteBucket) + .setStatus(OzoneManagerProtocolProtos.Status.OK) + .setDeleteBucketResponse( + DeleteBucketResponse.getDefaultInstance()).build(), + volumeName, bucketName); + + OmCompletedRequestInfo info = omBucketDeleteResponse.getCompletedRequestInfo(txIndex); + + assertEquals(txIndex, info.getTrxLogIndex()); + assertEquals(OzoneManagerProtocolProtos.Type.DeleteBucket, info.getCmdType()); + assertEquals(volumeName, info.getVolumeName()); + assertEquals(bucketName, info.getBucketName()); + assertTrue(info.getOpArgs() instanceof OmCompletedRequestInfo.OperationArgs.NoArgs); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java index aa152a5d2b76..15f205327087 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.file.Path; import java.util.ArrayList; @@ -34,6 +35,7 @@ import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; @@ -116,6 +118,40 @@ public void testAddToDBBatch() throws Exception { assertEquals(usedNamespace, keyValue.getValue().getUsedNamespace()); } + @Test + void testGetCompletedRequestInfo() { + long txIndex = 100L; + + String volumeName = UUID.randomUUID().toString(); + String keyName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + + OmKeyInfo omKeyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, + bucketName, OzoneFSUtils.addTrailingSlashIfNeeded(keyName), + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)).build(); + + OmBucketInfo omBucketInfo = TestOMResponseUtils.createBucket( + volumeName, bucketName); + + OMResponse omResponse = OMResponse.newBuilder().setCreateDirectoryResponse( + OzoneManagerProtocolProtos.CreateDirectoryResponse.getDefaultInstance()) + .setStatus(OzoneManagerProtocolProtos.Status.OK) + .setCmdType(OzoneManagerProtocolProtos.Type.CreateDirectory) + .build(); + + OMDirectoryCreateResponse omDirectoryCreateResponse = + new OMDirectoryCreateResponse(omResponse, omKeyInfo, + new ArrayList<>(), Result.SUCCESS, getBucketLayout(), omBucketInfo); + + OmCompletedRequestInfo info = omDirectoryCreateResponse.getCompletedRequestInfo(txIndex); + + assertEquals(txIndex, info.getTrxLogIndex()); + assertEquals(OzoneManagerProtocolProtos.Type.CreateDirectory, info.getCmdType()); + assertEquals(volumeName, info.getVolumeName()); + assertEquals(bucketName, info.getBucketName()); + assertTrue(info.getOpArgs() instanceof OmCompletedRequestInfo.OperationArgs.NoArgs); + } + public BucketLayout getBucketLayout() { return BucketLayout.DEFAULT; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseWithFSO.java index bc11a00a49f0..3427ff8c0081 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseWithFSO.java @@ -24,11 +24,13 @@ import java.util.ArrayList; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse; import org.apache.hadoop.ozone.om.response.key.TestOMKeyCreateResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; /** * Tests MKeyCreateResponse - prefix layout1. @@ -70,4 +72,14 @@ protected OMKeyCreateResponse getOmKeyCreateResponse(OmKeyInfo keyInfo, public BucketLayout getBucketLayout() { return BucketLayout.FILE_SYSTEM_OPTIMIZED; } + + @Override + public Type getCmdType() { + return Type.CreateFile; + } + + @Override + public OmCompletedRequestInfo.OperationArgs getCompletedRequestInfoArgs() { + return new OmCompletedRequestInfo.OperationArgs.CreateFileArgs(false, false); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponse.java index 1c2338f9d8a9..7fcf64107413 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponse.java @@ -30,6 +30,7 @@ import java.util.Map; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; @@ -130,6 +131,34 @@ public void testAddToDBBatchOnOverwrite() throws Exception { } + @Test + public void testGetCompletedRequestInfo() throws IOException { + long txIndex = 100L; + + OmKeyInfo omKeyInfo = getOmKeyInfo(); + + OzoneManagerProtocolProtos.OMResponse omResponse = + OzoneManagerProtocolProtos.OMResponse.newBuilder().setCommitKeyResponse( + OzoneManagerProtocolProtos.CommitKeyResponse.getDefaultInstance()) + .setStatus(OzoneManagerProtocolProtos.Status.OK) + .setCmdType(OzoneManagerProtocolProtos.Type.CommitKey) + .build(); + + String openKey = getOpenKeyName(); + String ozoneKey = getOzoneKey(); + + OMKeyCommitResponse omKeyCommitResponse = getOmKeyCommitResponse( + omKeyInfo, omResponse, openKey, ozoneKey, keysToDelete, false, null); + + OmCompletedRequestInfo info = omKeyCommitResponse.getCompletedRequestInfo(txIndex); + + assertEquals(txIndex, info.getTrxLogIndex()); + assertEquals(OzoneManagerProtocolProtos.Type.CommitKey, info.getCmdType()); + assertEquals(omKeyInfo.getVolumeName(), info.getVolumeName()); + assertEquals(omKeyInfo.getBucketName(), info.getBucketName()); + assertTrue(info.getOpArgs() instanceof OmCompletedRequestInfo.OperationArgs.NoArgs); + } + @Nonnull protected void addKeyToOpenKeyTable() throws Exception { OMRequestTestUtils.addKeyToTable(true, volumeName, bucketName, keyName, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponse.java index a44d955cb963..9a6ba40abd80 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponse.java @@ -17,12 +17,14 @@ package org.apache.hadoop.ozone.om.response.key; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import jakarta.annotation.Nonnull; import java.io.IOException; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateKeyResponse; @@ -38,6 +40,14 @@ protected long getVolumeId() throws IOException { return omMetadataManager.getVolumeId(volumeName); } + protected OzoneManagerProtocolProtos.Type getCmdType() { + return OzoneManagerProtocolProtos.Type.CreateKey; + } + + protected OmCompletedRequestInfo.OperationArgs getCompletedRequestInfoArgs() { + return new OmCompletedRequestInfo.OperationArgs.NoArgs(); + } + @Test public void testAddToDBBatch() throws Exception { @@ -93,6 +103,30 @@ public void testAddToDBBatchWithErrorResponse() throws Exception { } + @Test + public void testGetCompletedRequestInfo() throws IOException { + long txIndex = 100L; + OmKeyInfo omKeyInfo = getOmKeyInfo(); + + OMResponse omResponse = OMResponse.newBuilder().setCreateKeyResponse( + CreateKeyResponse.getDefaultInstance()) + .setStatus(OzoneManagerProtocolProtos.Status.OK) + .setCmdType(getCmdType()) + .build(); + + OMKeyCreateResponse omKeyCreateResponse = + getOmKeyCreateResponse(omKeyInfo, omBucketInfo, + omResponse); + + OmCompletedRequestInfo info = omKeyCreateResponse.getCompletedRequestInfo(txIndex); + + assertEquals(txIndex, info.getTrxLogIndex()); + assertEquals(getCmdType(), info.getCmdType()); + assertEquals(omKeyInfo.getVolumeName(), info.getVolumeName()); + assertEquals(omKeyInfo.getBucketName(), info.getBucketName()); + assertEquals(getCompletedRequestInfoArgs(), info.getOpArgs()); + } + @Nonnull protected OMKeyCreateResponse getOmKeyCreateResponse(OmKeyInfo keyInfo, OmBucketInfo bucketInfo, OMResponse response) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseWithFSO.java index 169d0b33727a..ff4fb24baa0e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseWithFSO.java @@ -29,12 +29,18 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; /** * Tests OMKeyCreateResponseWithFSO. */ public class TestOMKeyCreateResponseWithFSO extends TestOMKeyCreateResponse { + @Override + public Type getCmdType() { + return Type.CreateKey; + } + @Nonnull @Override protected String getOpenKeyName() throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponse.java index 33fcd137e66c..e9b992e17720 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponse.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om.response.key; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -29,6 +30,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; @@ -159,6 +161,34 @@ public void testAddToDBBatchWithErrorResponse() throws Exception { } + @Test + public void testGetCompletedRequestInfo() throws Exception { + long txIndex = 100L; + + String ozoneKey = addKeyToTable(); + OmKeyInfo omKeyInfo = omMetadataManager + .getKeyTable(getBucketLayout()).get(ozoneKey); + + OzoneManagerProtocolProtos.OMResponse omResponse = + OzoneManagerProtocolProtos.OMResponse.newBuilder().setDeleteKeyResponse( + OzoneManagerProtocolProtos.DeleteKeyResponse.getDefaultInstance()) + .setStatus(OzoneManagerProtocolProtos.Status.OK) + .setCmdType(OzoneManagerProtocolProtos.Type.DeleteKey) + .build(); + + OMKeyDeleteResponse omKeyDeleteResponse = getOmKeyDeleteResponse(omKeyInfo, + omResponse); + + OmCompletedRequestInfo info = omKeyDeleteResponse.getCompletedRequestInfo(txIndex); + + assertEquals(txIndex, info.getTrxLogIndex()); + assertEquals(OzoneManagerProtocolProtos.Type.DeleteKey, info.getCmdType()); + assertEquals(omKeyInfo.getVolumeName(), info.getVolumeName()); + assertEquals(omKeyInfo.getBucketName(), info.getBucketName()); + assertEquals(OmCompletedRequestInfo.OperationArgs.NoArgs.class, + info.getOpArgs().getClass()); + } + protected String addKeyToTable() throws Exception { String ozoneKey = omMetadataManager.getOzoneKey(volumeName, bucketName, keyName); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponse.java index aa9e69466959..c127bec56883 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponse.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -151,6 +152,40 @@ public void testAddToDBBatchWithErrorResponse() throws Exception { } } + @Test + public void testGetCompletedRequestInfo() throws Exception { + long txIndex = 100L; + + OMResponse omResponse = + OMResponse.newBuilder().setRenameKeyResponse( + OzoneManagerProtocolProtos.RenameKeyResponse.getDefaultInstance()) + .setStatus(OzoneManagerProtocolProtos.Status.OK) + .setCmdType(OzoneManagerProtocolProtos.Type.RenameKey) + .build(); + + String toKeyName = UUID.randomUUID().toString(); + OmKeyInfo toKeyInfo = getOmKeyInfo(toKeyName); + OmKeyInfo fromKeyInfo = getOmKeyInfo(toKeyInfo, keyName); + String dbFromKey = addKeyToTable(fromKeyInfo); + String dbToKey = getDBKeyName(toKeyInfo); + + OMKeyRenameResponse omKeyRenameResponse = + getOMKeyRenameResponse(omResponse, fromKeyInfo, toKeyInfo); + + OmCompletedRequestInfo info = omKeyRenameResponse.getCompletedRequestInfo(txIndex); + + assertEquals(txIndex, info.getTrxLogIndex()); + assertEquals(OzoneManagerProtocolProtos.Type.RenameKey, info.getCmdType()); + assertEquals(fromKeyInfo.getVolumeName(), info.getVolumeName()); + assertEquals(fromKeyInfo.getBucketName(), info.getBucketName()); + assertEquals(fromKeyInfo.getKeyName(), info.getKeyName()); + + assertEquals(OmCompletedRequestInfo.OperationArgs.RenameKeyArgs.class, + info.getOpArgs().getClass()); + assertEquals(new OmCompletedRequestInfo.OperationArgs.RenameKeyArgs(toKeyInfo.getKeyName()), + info.getOpArgs()); + } + protected OmKeyInfo getOmKeyInfo(String keyName) { return OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName, replicationConfig).build(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponseWithFSO.java index a1d14fadad30..20db043d9d35 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponseWithFSO.java @@ -73,7 +73,8 @@ protected OMKeyRenameResponse getOMKeyRenameResponse(OMResponse response, createParent(); return new OMKeyRenameResponseWithFSO(response, getDBKeyName(fromKeyInfo), getDBKeyName(toKeyInfo), fromKeyParent, toKeyParent, toKeyInfo, - bucketInfo, false, getBucketLayout()); + bucketInfo, false, getBucketLayout(), + fromKeyInfo.getKeyName(), toKeyInfo.getKeyName()); } protected void createParent() {