Skip to content

Commit

Permalink
rdar://134867479 (Client Side Encryption is missing from Iceberg Rest…
Browse files Browse the repository at this point in the history
… Catalog) (apache#1312)

* (apple-internal) Support encryption/decryption in REST Catalog

---------

Co-authored-by: huanghsiang_cheng <[email protected]>
  • Loading branch information
2 people authored and GitHub Enterprise committed Aug 28, 2024
1 parent 6f52a07 commit 90ef7e9
Show file tree
Hide file tree
Showing 6 changed files with 532 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableCommit;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.encryption.EncryptionManagerFactory;
import org.apache.iceberg.exceptions.AlreadyExistsException;
import org.apache.iceberg.exceptions.NoSuchNamespaceException;
import org.apache.iceberg.exceptions.NoSuchTableException;
Expand Down Expand Up @@ -134,6 +135,7 @@ public class RESTSessionCatalog extends BaseViewSessionCatalog
private SnapshotMode snapshotMode = null;
private Object conf = null;
private FileIO io = null;
private EncryptionManagerFactory encryptionManagerFactory = null;
private MetricsReporter reporter = null;
private boolean reportingViaRestEnabled;
private CloseableGroup closeables = null;
Expand Down Expand Up @@ -229,10 +231,12 @@ public void initialize(String name, Map<String, String> unresolved) {
}

this.io = newFileIO(SessionContext.createEmpty(), mergedProps);
this.encryptionManagerFactory = CatalogUtil.loadEncryptionManagerFactory(mergedProps);

this.fileIOCloser = newFileIOCloser();
this.closeables = new CloseableGroup();
this.closeables.addCloseable(this.io);
this.closeables.addCloseable(this.encryptionManagerFactory);
this.closeables.addCloseable(this.client);
this.closeables.setSuppressCloseFailure(true);

Expand Down Expand Up @@ -396,6 +400,7 @@ public Table loadTable(SessionContext context, TableIdentifier identifier) {
paths.table(finalIdentifier),
session::headers,
tableFileIO(context, response.config()),
encryptionManagerFactory,
tableMetadata);

trackFileIO(ops);
Expand Down Expand Up @@ -469,6 +474,7 @@ public Table registerTable(
paths.table(ident),
session::headers,
tableFileIO(context, response.config()),
encryptionManagerFactory,
response.tableMetadata());

trackFileIO(ops);
Expand Down Expand Up @@ -688,6 +694,7 @@ public Table create() {
paths.table(ident),
session::headers,
tableFileIO(context, response.config()),
encryptionManagerFactory,
response.tableMetadata());

trackFileIO(ops);
Expand All @@ -710,6 +717,7 @@ public Transaction createTransaction() {
paths.table(ident),
session::headers,
tableFileIO(context, response.config()),
encryptionManagerFactory,
RESTTableOperations.UpdateType.CREATE,
createChanges(meta),
meta);
Expand Down Expand Up @@ -774,6 +782,7 @@ public Transaction replaceTransaction() {
paths.table(ident),
session::headers,
tableFileIO(context, response.config()),
encryptionManagerFactory,
RESTTableOperations.UpdateType.REPLACE,
changes.build(),
base);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
import org.apache.iceberg.UpdateRequirement;
import org.apache.iceberg.UpdateRequirements;
import org.apache.iceberg.encryption.EncryptionManager;
import org.apache.iceberg.encryption.EncryptionManagerFactory;
import org.apache.iceberg.encryption.PlaintextEncryptionManager;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.LocationProvider;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
Expand All @@ -54,6 +56,7 @@ enum UpdateType {
private final String path;
private final Supplier<Map<String, String>> headers;
private final FileIO io;
private final EncryptionManagerFactory encryptionManagerFactory;
private final List<MetadataUpdate> createChanges;
private final TableMetadata replaceBase;
private UpdateType updateType;
Expand All @@ -64,22 +67,33 @@ enum UpdateType {
String path,
Supplier<Map<String, String>> headers,
FileIO io,
EncryptionManagerFactory encryptionManagerFactory,
TableMetadata current) {
this(client, path, headers, io, UpdateType.SIMPLE, Lists.newArrayList(), current);
this(
client,
path,
headers,
io,
encryptionManagerFactory,
UpdateType.SIMPLE,
Lists.newArrayList(),
current);
}

RESTTableOperations(
RESTClient client,
String path,
Supplier<Map<String, String>> headers,
FileIO io,
EncryptionManagerFactory encryptionManagerFactory,
UpdateType updateType,
List<MetadataUpdate> createChanges,
TableMetadata current) {
this.client = client;
this.path = path;
this.headers = headers;
this.io = io;
this.encryptionManagerFactory = encryptionManagerFactory;
this.updateType = updateType;
this.createChanges = createChanges;
this.replaceBase = current;
Expand Down Expand Up @@ -194,6 +208,16 @@ public LocationProvider locationProvider() {
return LocationProviders.locationsFor(current().location(), current().properties());
}

@Override
public EncryptionManager encryption() {
TableMetadata metadata = current();
if (null != metadata) {
return encryptionManagerFactory.create(metadata);
} else {
return PlaintextEncryptionManager.instance();
}
}

@Override
public TableOperations temp(TableMetadata uncommittedMetadata) {
return new TableOperations() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.iceberg.encryption;

import static org.apache.iceberg.Files.localOutput;

import java.io.File;
import org.apache.iceberg.io.InputFile;
import org.apache.iceberg.io.OutputFile;
import org.apache.iceberg.io.PositionOutputStream;

public class EncryptedLocalOutputFile implements OutputFile, NativelyEncryptedFile {
private OutputFile localOutputFile;
private NativeFileCryptoParameters nativeEncryptionParameters;

public EncryptedLocalOutputFile(File file) {
localOutputFile = localOutput(file);
}

@Override
public PositionOutputStream create() {
return localOutputFile.create();
}

@Override
public PositionOutputStream createOrOverwrite() {
return localOutputFile.createOrOverwrite();
}

@Override
public String location() {
return localOutputFile.location();
}

@Override
public InputFile toInputFile() {
return localOutputFile.toInputFile();
}

@Override
public NativeFileCryptoParameters nativeCryptoParameters() {
return nativeEncryptionParameters;
}

@Override
public void setNativeCryptoParameters(NativeFileCryptoParameters nativeCryptoParameters) {
this.nativeEncryptionParameters = nativeCryptoParameters;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.iceberg.encryption.kms;

import java.nio.charset.StandardCharsets;
import java.util.Map;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;

public class UnitestKMS extends MemoryMockKMS {
public static final String MASTER_KEY_NAME1 = "keyA";
public static final byte[] MASTER_KEY1 = "0123456789012345".getBytes(StandardCharsets.UTF_8);
public static final String MASTER_KEY_NAME2 = "keyB";
public static final byte[] MASTER_KEY2 = "1123456789012345".getBytes(StandardCharsets.UTF_8);

@Override
public void initialize(Map<String, String> properties) {
masterKeys =
ImmutableMap.of(
MASTER_KEY_NAME1, MASTER_KEY1,
MASTER_KEY_NAME2, MASTER_KEY2);
}
}
Loading

0 comments on commit 90ef7e9

Please sign in to comment.