Skip to content

Commit

Permalink
DDF-3660 Update CDM in place monitoring to CreateRequests (#3033)
Browse files Browse the repository at this point in the history 
* Update CDM in-place monitoring to send create requests instead of create storage requests

* Update LocalResourceRetriever to account for derived content

* Remove EventfulFileWrapper and change DurableFileEndpoint to File type
  • Loading branch information
@peterhuffer @clockard
peterhuffer authored and clockard committed Mar 8, 2018
1 parent 3b104fd commit 1aefecf
Show file tree
Hide file tree
Showing 38 changed files with 1,217 additions and 486 deletions.
3 changes: 2 additions & 1 deletion catalog/core/catalog-core-api/src/main/java/ddf/catalog/data/types/Core.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,8 @@ public interface Core {

/**
* {@link ddf.catalog.data.Attribute} name for accessing the resource download URL for the product
* this {@link ddf.catalog.data.Metacard} represents. <br>
* this {@link ddf.catalog.data.Metacard} represents. Typically, the framework will write this
* attribute based on the {@link Core#RESOURCE_URI}, and it should not be set directly. <br>
*/
String RESOURCE_DOWNLOAD_URL = "resource-download-url";

Expand Down
4 changes: 2 additions & 2 deletions catalog/core/catalog-core-camelcomponent/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,12 +182,12 @@
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.66</minimum>
<minimum>0.63</minimum>
</limit>
<limit>
<counter>BRANCH</counter>
<value>COVEREDRATIO</value>
<minimum>0.53</minimum>
<minimum>0.50</minimum>
</limit>
<limit>
<counter>COMPLEXITY</counter>
Expand Down
22 changes: 14 additions & 8 deletions ...talog-core-camelcomponent/src/main/java/ddf/camel/component/catalog/CatalogComponent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

import ddf.catalog.CatalogFramework;
import ddf.catalog.transform.CatalogTransformerException;
import ddf.mime.MimeTypeMapper;
import ddf.mime.MimeTypeToTransformerMapper;
import java.util.Map;
import org.apache.camel.Endpoint;
Expand All @@ -39,14 +40,16 @@ public class CatalogComponent extends DefaultComponent {
/** The name of the scheme this custom Camel component resolves to. */
public static final String NAME = "catalog";

private static final transient Logger LOGGER = LoggerFactory.getLogger(CatalogComponent.class);
private static final Logger LOGGER = LoggerFactory.getLogger(CatalogComponent.class);

private BundleContext bundleContext;

private MimeTypeToTransformerMapper mimeTypeToTransformerMapper;

private CatalogFramework catalogFramework;

private MimeTypeMapper mimeTypeMapper;

public CatalogComponent() {
super();
LOGGER.debug("INSIDE CatalogComponent constructor");
Expand All @@ -60,28 +63,27 @@ public CatalogComponent() {
*/
protected Endpoint createEndpoint(String uri, String remaining, Map<String, Object> parameters)
throws CatalogTransformerException {
LOGGER.debug("ENTERING: createEndpoint");
LOGGER.trace("ENTERING: createEndpoint");

LOGGER.debug("uri = {}, remaining = {}", uri, remaining);
LOGGER.debug("parameters = {}", parameters);
LOGGER.debug(
"CatalogEndpoint: uri = {}, remaining = {}, parameters = {}", uri, remaining, parameters);

String contextPath = remaining;
String transformerId = getAndRemoveParameter(parameters, ID_PARAMETER, String.class);

String mimeType = getAndRemoveParameter(parameters, MIME_TYPE_PARAMETER, String.class);

LOGGER.debug("transformerId = {}", transformerId);

Endpoint endpoint =
new CatalogEndpoint(uri, this, transformerId, mimeType, contextPath, catalogFramework);
new CatalogEndpoint(
uri, this, transformerId, mimeType, remaining, catalogFramework, mimeTypeMapper);
try {
setProperties(endpoint, parameters);
} catch (Exception e) {
throw new CatalogTransformerException("Failed to create transformer endpoint", e);
}

LOGGER.debug("EXITING: createEndpoint");

LOGGER.trace("EXITING: createEndpoint");
return endpoint;
}

Expand Down Expand Up @@ -132,4 +134,8 @@ public void setMimeTypeToTransformerMapper(
public void setCatalogFramework(CatalogFramework catalogFramework) {
this.catalogFramework = catalogFramework;
}

public void setMimeTypeMapper(MimeTypeMapper mimeTypeMapper) {
this.mimeTypeMapper = mimeTypeMapper;
}
}
13 changes: 11 additions & 2 deletions ...atalog-core-camelcomponent/src/main/java/ddf/camel/component/catalog/CatalogEndpoint.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import ddf.camel.component.catalog.queryresponsetransformer.QueryResponseTransformerConsumer;
import ddf.camel.component.catalog.queryresponsetransformer.QueryResponseTransformerProducer;
import ddf.catalog.CatalogFramework;
import ddf.mime.MimeTypeMapper;
import org.apache.camel.Consumer;
import org.apache.camel.ExchangePattern;
import org.apache.camel.MultipleConsumersSupport;
Expand All @@ -38,7 +39,7 @@
* @author [email protected]
*/
public class CatalogEndpoint extends DefaultEndpoint implements MultipleConsumersSupport {
private static final transient Logger LOGGER = LoggerFactory.getLogger(CatalogEndpoint.class);
private static final Logger LOGGER = LoggerFactory.getLogger(CatalogEndpoint.class);

private static final String INPUT_TRANSFORMER = "inputtransformer";

Expand All @@ -58,6 +59,8 @@ public class CatalogEndpoint extends DefaultEndpoint implements MultipleConsumer

private CatalogFramework catalogFramework;

private MimeTypeMapper mimeTypeMapper;

/**
* Constructs a CatalogEndpoint for the specified custom <code>catalog</code> component.
*
Expand All @@ -76,14 +79,16 @@ public CatalogEndpoint(
String transformerId,
String mimeType,
String contextPath,
CatalogFramework catalogFramework) {
CatalogFramework catalogFramework,
MimeTypeMapper mimeTypeMapper) {
super(uri, component);
LOGGER.debug(
"INSIDE CamelCatalogEndpoint(uri, component, transformerId, contextPath, catalogFramework) constructor");
this.transformerId = transformerId;
this.mimeType = mimeType;
this.contextPath = contextPath;
this.catalogFramework = catalogFramework;
this.mimeTypeMapper = mimeTypeMapper;
setSynchronous(true);
}

Expand Down Expand Up @@ -178,6 +183,10 @@ public String getMimeType() {
return mimeType;
}

public MimeTypeMapper getMimeTypeMapper() {
return mimeTypeMapper;
}

/*
* (non-Javadoc)
*
Expand Down
2 changes: 1 addition & 1 deletion ...re-camelcomponent/src/main/java/ddf/camel/component/catalog/content/ContentComponent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public class ContentComponent extends DefaultComponent {
/** The name of the scheme this custom Camel component resolves to. */
public static final String NAME = "content";

private static final transient Logger LOGGER = LoggerFactory.getLogger(ContentComponent.class);
private static final Logger LOGGER = LoggerFactory.getLogger(ContentComponent.class);

private CatalogFramework catalogFramework;

Expand Down
51 changes: 24 additions & 27 deletions ...camelcomponent/src/main/java/ddf/camel/component/catalog/framework/FrameworkProducer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@
*/
public class FrameworkProducer extends DefaultProducer {

private static final transient Logger LOGGER = LoggerFactory.getLogger(FrameworkProducer.class);
private static final Logger LOGGER = LoggerFactory.getLogger(FrameworkProducer.class);

private static final String CREATE_OPERATION = "CREATE";

Expand All @@ -99,6 +99,8 @@ public class FrameworkProducer extends DefaultProducer {

private static final String OPERATION_HEADER_KEY = "operation";

private static final String OBJECT_IS_NULL_MSG = "Catalog response object is null";

private CatalogFramework catalogFramework;

/**
Expand All @@ -118,12 +120,12 @@ public void process(Exchange exchange) throws FrameworkProducerException {
LOGGER.debug("Entering process method");

final Object operationValueObj = exchange.getIn().getHeader(OPERATION_HEADER_KEY);
String operation = null;
String operation;

if (operationValueObj == null) {
exchange.getIn().setBody(new ArrayList<Metacard>());

throw new FrameworkProducerException("Missing expected header!");
throw new FrameworkProducerException(
String.format("Missing expected [%s] header!", OPERATION_HEADER_KEY));
}

operation = operationValueObj.toString();
Expand All @@ -145,12 +147,9 @@ public void process(Exchange exchange) throws FrameworkProducerException {
} catch (ClassCastException cce) {
exchange.getIn().setBody(new ArrayList<Metacard>());
LOGGER.debug("Received a non-String as the operation type");
} catch (SourceUnavailableException sue) {
exchange.getIn().setBody(new ArrayList<Metacard>());
LOGGER.debug("Exception cataloging metacards", sue);
} catch (IngestException ie) {
} catch (SourceUnavailableException | IngestException e) {
exchange.getIn().setBody(new ArrayList<Metacard>());
LOGGER.debug("Exception cataloging metacards", ie);
LOGGER.debug("Exception cataloging metacards", e);
}
}

Expand Down Expand Up @@ -365,14 +364,12 @@ private void delete(final Exchange exchange)
* @return true if the list is not empty and has valid types inside, else false.
*/
private boolean validateList(List<?> list, Class<?> cls) {
if (list.size() == 0) {
if (list.isEmpty()) {
LOGGER.debug("No Metacard or Metacard IDs to process");
return false;
}

for (int i = 0; i < list.size(); i++) {
final Object o = list.get(i);

for (final Object o : list) {
if (!cls.isInstance(o)) {
LOGGER.debug("Received a list of non-{} objects", cls.getName());
return false;
Expand All @@ -390,14 +387,14 @@ private boolean validateList(List<?> list, Class<?> cls) {
*/
private void processCatalogResponse(final CreateResponse response, final Exchange exchange) {
if (response == null) {
LOGGER.debug("Catalog response object is null");
exchange.getIn().setBody((List) (new ArrayList<Metacard>()));
LOGGER.debug(OBJECT_IS_NULL_MSG);
exchange.getIn().setBody(new ArrayList<Metacard>());
return;
}

if (response.getCreatedMetacards() == null) {
LOGGER.debug("No Metacards created by catalog framework");
exchange.getIn().setBody((List) (new ArrayList<Metacard>()));
exchange.getIn().setBody(new ArrayList<Metacard>());
return;
}

Expand All @@ -412,14 +409,14 @@ private void processCatalogResponse(final CreateResponse response, final Exchang
*/
private void processCatalogResponse(final UpdateResponse response, final Exchange exchange) {
if (response == null) {
LOGGER.debug("Catalog response object is null");
exchange.getIn().setBody((List) (new ArrayList<Metacard>()));
LOGGER.debug(OBJECT_IS_NULL_MSG);
exchange.getIn().setBody(new ArrayList<Metacard>());
return;
}

if (response.getUpdatedMetacards() == null) {
LOGGER.debug("No Metacards updated by catalog framework");
exchange.getIn().setBody((List) (new ArrayList<Metacard>()));
exchange.getIn().setBody(new ArrayList<Metacard>());
return;
}

Expand All @@ -434,14 +431,14 @@ private void processCatalogResponse(final UpdateResponse response, final Exchang
*/
private void processCatalogResponse(final DeleteResponse response, final Exchange exchange) {
if (response == null) {
LOGGER.debug("Catalog response object is null");
exchange.getIn().setBody((List) (new ArrayList<Metacard>()));
LOGGER.debug(OBJECT_IS_NULL_MSG);
exchange.getIn().setBody(new ArrayList<Metacard>());
return;
}

if (response.getDeletedMetacards() == null) {
LOGGER.debug("No Metacards deleted by catalog framework");
exchange.getIn().setBody((List) (new ArrayList<Metacard>()));
exchange.getIn().setBody(new ArrayList<Metacard>());
return;
}

Expand All @@ -455,7 +452,7 @@ private void processCatalogResponse(final DeleteResponse response, final Exchang
* @return {@link java.util.List} of {@link String} representing Metacard IDs
*/
private List<String> readBodyDataAsMetacardIds(final Exchange exchange) {
List<String> metacardIdsToBeProcessed = new ArrayList<String>();
List<String> metacardIdsToBeProcessed = new ArrayList<>();

try {
if (exchange.getIn().getBody() == null) {
Expand All @@ -478,15 +475,15 @@ private List<String> readBodyDataAsMetacardIds(final Exchange exchange) {
final String metacardIdToBeProcessed = exchange.getIn().getBody(String.class);

if (metacardIdToBeProcessed != null) {
metacardIdsToBeProcessed = new ArrayList<String>();
metacardIdsToBeProcessed = new ArrayList<>();
metacardIdsToBeProcessed.add(metacardIdToBeProcessed);
LOGGER.debug("Successfully read in body data as String");
return metacardIdsToBeProcessed;
}

// if we get here, we neither had String or List<?>, so set a
// default list
metacardIdsToBeProcessed = new ArrayList<String>();
metacardIdsToBeProcessed = new ArrayList<>();
} catch (TypeConversionException tce1) {
LOGGER.debug("Invalid message body. Expected either String or List<String>", tce1);
}
Expand All @@ -501,7 +498,7 @@ private List<String> readBodyDataAsMetacardIds(final Exchange exchange) {
* @return {@link java.util.List} of Metacard objects
*/
private List<Metacard> readBodyDataAsMetacards(final Exchange exchange) {
List<Metacard> metacardsToProcess = new ArrayList<Metacard>();
List<Metacard> metacardsToProcess = new ArrayList<>();

try {
if (exchange.getIn().getBody() == null) {
Expand All @@ -526,7 +523,7 @@ private List<Metacard> readBodyDataAsMetacards(final Exchange exchange) {
metacardsToProcess = exchange.getIn().getBody(List.class);
if (metacardsToProcess == null) {
LOGGER.debug("Problem reading in body data as List<?>");
metacardsToProcess = new ArrayList<Metacard>();
metacardsToProcess = new ArrayList<>();
return metacardsToProcess;
}
LOGGER.debug("Successfully read in body data as List<?>");
Expand Down
Loading

0 comments on commit 1aefecf

Please sign in to comment.