Skip to content

Commit

Permalink
Merge pull request #97 from oracle/release_2023-11-07
Browse files Browse the repository at this point in the history
Releasing version 3.3.4.1.3.0
  • Loading branch information
yanhaizhongyu authored Nov 7, 2023
2 parents d8c7306 + a8bb9a3 commit 83bcd77
Show file tree
Hide file tree
Showing 29 changed files with 1,616 additions and 60 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](http://keepachangelog.com/).

## 3.3.4.1.3.0 - 2023-11-07
### Added
- Added observability feature that generates metrics pertaining to operations conducted through the connector, such as reading, writing, and deleting data.

## 3.3.4.1.2.1 - 2023-10-20
### Added
- Add multi-region support for the same configuration. This feature can be enabled by setting `fs.oci.client.multiregion.enabled` to true. Once enabled, the user has the option to append a region code or ID after the namespace in the `oci://<bucket>@<namespace>.<region>/file` format. This action will result in the creation of a dedicated BmcFilesystem instance for the specified region. Your applications can then make use of different URIs to create BmcFilesystem instances, each directed towards distinct endpoints.
Expand Down
6 changes: 3 additions & 3 deletions hdfs-addons/hdfs-smartparquet/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
<parent>
<artifactId>oci-hdfs-addons</artifactId>
<groupId>com.oracle.oci.sdk</groupId>
<version>3.3.4.1.2.1</version>
<version>3.3.4.1.3.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down Expand Up @@ -149,7 +149,7 @@
<dependency>
<groupId>com.oracle.oci.sdk</groupId>
<artifactId>oci-hdfs-connector</artifactId>
<version>3.3.4.1.2.1</version>
<version>3.3.4.1.3.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
Expand Down Expand Up @@ -251,7 +251,7 @@
<dependency>
<groupId>com.oracle.oci.sdk</groupId>
<artifactId>oci-hdfs-connector</artifactId>
<version>3.3.4.1.2.1</version>
<version>3.3.4.1.3.0</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
Expand Down
2 changes: 1 addition & 1 deletion hdfs-addons/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
<parent>
<artifactId>oci-hdfs</artifactId>
<groupId>com.oracle.oci.sdk</groupId>
<version>3.3.4.1.2.1</version>
<version>3.3.4.1.3.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
7 changes: 6 additions & 1 deletion hdfs-connector/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
<parent>
<groupId>com.oracle.oci.sdk</groupId>
<artifactId>oci-hdfs</artifactId>
<version>3.3.4.1.2.1</version>
<version>3.3.4.1.3.0</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down Expand Up @@ -335,6 +335,11 @@
<artifactId>oci-java-sdk-objectstorage-generated</artifactId>
<version>${oci.java.sdk.version}</version>
</dependency>
<dependency>
<groupId>com.oracle.oci.sdk</groupId>
<artifactId>oci-java-sdk-monitoring</artifactId>
<version>${oci.java.sdk.version}</version>
</dependency>
<dependency>
<groupId>com.oracle.oci.sdk</groupId>
<artifactId>oci-java-sdk-common</artifactId>
Expand Down
14 changes: 14 additions & 0 deletions hdfs-connector/src/main/java/com/oracle/bmc/hdfs/BmcConstants.java
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,20 @@ public final class BmcConstants {

public static final String RECURSIVE_DIR_LISTING_FETCH_SIZE_KEY = "fs.oci.dir.recursive.list.fetch.size";

/**
* Following are the properties related to OCI monitoring for the connector
*/
public static final String OCI_MONITORING_CONSUMER_PLUGINS_KEY = "fs.oci.mon.consumer.plugins";
public static final String OCI_MON_TELEMETRY_INGESTION_ENDPOINT_KEY = "fs.oci.mon.telemetry.ingestion.endpoint";
public static final String OCI_MON_COMPARTMENT_OCID_KEY = "fs.oci.mon.compartment.ocid";
public static final String OCI_MON_GROUPING_CLUSTER_ID_KEY = "fs.oci.mon.grouping.cluster.id";
public static final String OCI_MON_RG_NAME_KEY = "fs.oci.mon.rg.name";
public static final String OCI_MON_NS_NAME_KEY = "fs.oci.mon.ns.name";
public static final String OCI_MON_BUCKET_LEVEL_ENABLED_KEY = "fs.oci.mon.bucket.level.enabled";
public static final String OCI_MON_EMIT_THREAD_POLL_INTERVAL_SECONDS_KEY
= "fs.oci.emit.thread.poll.interval.seconds";
public static final String OCI_MON_MAX_BACKLOG_BEFORE_DROP_KEY = "fs.oci.mon.max.backlog.before.drop";

/**
* This class contains constants with deprecated values. The HDFS connector will first try the current values
* in {@link com.oracle.bmc.hdfs.BmcConstants}.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import com.oracle.bmc.auth.BasicAuthenticationDetailsProvider;
import com.oracle.bmc.hdfs.caching.StrongConsistencyPolicy;
import com.oracle.bmc.hdfs.monitoring.OCIMonitorConsumerPlugin;
import com.oracle.bmc.objectstorage.ObjectStorage;

import com.oracle.bmc.ClientConfiguration;
Expand Down Expand Up @@ -510,7 +511,70 @@ public enum BmcProperties {
* to be listed in one call to OSS Service (essentially the page size of the OSS listing API).
* See {@link BmcConstants#RECURSIVE_DIR_LISTING_FETCH_SIZE_KEY} for config key name.
*/
RECURSIVE_DIR_LISTING_FETCH_SIZE(RECURSIVE_DIR_LISTING_FETCH_SIZE_KEY, 1000)
RECURSIVE_DIR_LISTING_FETCH_SIZE(RECURSIVE_DIR_LISTING_FETCH_SIZE_KEY, 1000),

/**
* (String, optional) This defines what plugins are available to consume the metrics collected by the OCI
* monitoring framework. This has to be the fully qualified classname of the class extending the
* {@link OCIMonitorConsumerPlugin} class. Comma separated values are accepted.
* for multiple plugins.
* See {@link BmcConstants#OCI_MON_TELEMETRY_INGESTION_ENDPOINT_KEY} for config key name.
*/
OCI_MONITORING_CONSUMER_PLUGINS(OCI_MONITORING_CONSUMER_PLUGINS_KEY,
"com.oracle.bmc.hdfs.monitoring.OCIMonitorPlugin"),

/**
* (String, optional) The endpoint for telemetry ingestion. This is required if OCI monitoring is enabled.
* A typical endpoint is like the following: https://telemetry-ingestion.us-ashburn-1.oraclecloud.com
* See {@link BmcConstants#OCI_MON_TELEMETRY_INGESTION_ENDPOINT_KEY} for config key name.
*/
OCI_MON_TELEMETRY_INGESTION_ENDPOINT(OCI_MON_TELEMETRY_INGESTION_ENDPOINT_KEY, null),

/**
* (String, optional) The compartment OCID to which the metrics will be emitted. This compartment should
* have the right Identity policies setup for receiving monitoring information. This field is mandatory only if
* the OCI monitoring is enabled.
* See {@link BmcConstants#OCI_MON_COMPARTMENT_OCID_KEY} for config key name.
*/
OCI_MON_COMPARTMENT_OCID(OCI_MON_COMPARTMENT_OCID_KEY, null),

/**
* (String, optional) This is the unique identifier used to group analytics values. For example it could be the
* ID of the HDFS cluster. All the metrics belonging to this ID are grouped together on OCI monitoring no matter
* where they are emitted from. This field is mandatory only if OCI monitoring is enabled.
* See {@link BmcConstants#OCI_MON_GROUPING_CLUSTER_ID_KEY} for config key name.
*/
OCI_MON_GROUPING_CLUSTER_ID(OCI_MON_GROUPING_CLUSTER_ID_KEY, null),

/**
* (String, optional) This is the resource group name on the OCI monitoring.
* See {@link BmcConstants#OCI_MON_RG_NAME_KEY} for config key name.
*/
OCI_MON_RG_NAME(OCI_MON_RG_NAME_KEY, "apicallstest"),

/**
* (String, optional) This is the namespace name on the OCI monitoring.
* See {@link BmcConstants#OCI_MON_NS_NAME_KEY} for config key name.
*/
OCI_MON_NS_NAME(OCI_MON_NS_NAME_KEY, "ocihdfsconnector"),

/**
* (boolean, optional) This field determines if the stats are emitted with bucket name as a dimension or not.
* See {@link BmcConstants#OCI_MON_BUCKET_LEVEL_ENABLED_KEY} for config key name.
*/
OCI_MON_BUCKET_LEVEL_ENABLED(OCI_MON_BUCKET_LEVEL_ENABLED_KEY, true),

/**
* (Integer, optional) The sleep time for the monitor thread in between runs.
* See {@link BmcConstants#OCI_MON_EMIT_THREAD_POLL_INTERVAL_SECONDS_KEY} for config key name.
*/
OCI_MON_EMIT_THREAD_POLL_INTERVAL_SECONDS(OCI_MON_EMIT_THREAD_POLL_INTERVAL_SECONDS_KEY, 2),

/**
* (Integer, optional) The maximum backlog that will be allowed in the list before no-admit.
* See {@link BmcConstants#OCI_MON_MAX_BACKLOG_BEFORE_DROP_KEY} for config key name.
*/
OCI_MON_MAX_BACKLOG_BEFORE_DROP(OCI_MON_MAX_BACKLOG_BEFORE_DROP_KEY, 100000)
;

@Getter private final String propertyName;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package com.oracle.bmc.hdfs.monitoring;

import com.oracle.bmc.hdfs.store.BmcPropertyAccessor;
import lombok.extern.slf4j.Slf4j;

import java.util.ArrayList;
import java.util.List;

@Slf4j
public class DummyOCIMonitoringPlugin extends OCIMonitorConsumerPlugin {

private static DummyOCIMonitoringPlugin INSTANCE;

private List<OCIMetric> ociMetricList = new ArrayList<>();

public static DummyOCIMonitoringPlugin getInstance() {
return INSTANCE;
}

public DummyOCIMonitoringPlugin(BmcPropertyAccessor propertyAccessor, String bucketName,
String monitoringGroupingID, String namespaceName) {
super(propertyAccessor, bucketName, monitoringGroupingID, namespaceName);
LOG.debug("DummyOCIMonitoringPlugin constructor {} {} {} {}",propertyAccessor, bucketName,
monitoringGroupingID, namespaceName);
}

@Override
public void accept(OCIMetric ociMetric) {
INSTANCE = this;

try {

// Hold on to 100 metric objects for verification in tests and drop them.
if (ociMetricList.size() > 100) {
ociMetricList = new ArrayList<>();
}
ociMetricList.add(ociMetric);

LOG.debug("Accepting {} inside accept method of DummyOCIMonitoringPlugin.", ociMetric);
} catch (Exception e) {
// Ignore this exception
}
}

public List<OCIMetric> getMetrics() {
return ociMetricList;
}

@Override
public void shutdown() {
LOG.debug("Inside shutdown method of DummyOCIMonitoringPlugin.");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package com.oracle.bmc.hdfs.monitoring;

import com.oracle.bmc.model.BmcException;

public class OCIMetric {
/**
* The time in milliseconds (epoch) when the metric was recorded.
*/
private final long recordedTime;
/**
* The overall time taken by the operation to complete/error-out in milliseconds.
*/
private final double overallTime;
/**
* The operation key. This will be one of {"LIST", "HEAD", "WRITE", "READ", "DELETE", "RENAME"}
*/
private final String key;
/**
* The boolean error indicates whether the operation errored out.
*/
private final boolean error;
/**
* If the operation errored out, then the errorStatusCode is populated with the status code of the
* error when communicating with OCI.
*/
private final int errorStatusCode;
/**
* The target OCI bucket where the operation was attempted to.
*/
private final String bucketName;

public OCIMetric(double overallTime, String key, Exception e, String bucketName) {
this.recordedTime = System.currentTimeMillis();
this.overallTime = overallTime;
this.key = key;
this.bucketName = bucketName;

int errStatusCode = 0;
if (e != null) {
this.error = true;
if (e instanceof BmcException) {
errStatusCode = ((BmcException) e).getStatusCode();
}
} else {
this.error = false;
}
this.errorStatusCode = errStatusCode;
}

public String getKey() {
return key;
}

public double getOverallTime() {
return overallTime;
}

public boolean isError() {
return error;
}

public long getRecordedTime() {
return recordedTime;
}

public String getBucketName() { return bucketName; }

public int getErrorStatusCode() { return errorStatusCode; }

@Override
public String toString() {
return "OCIMetric{" +
"recordedTime=" + recordedTime +
", overallTime=" + overallTime +
", key='" + key + '\'' +
", error=" + error +
", bucketName=" + bucketName +
", errorStatusCode=" + errorStatusCode +
'}';
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package com.oracle.bmc.hdfs.monitoring;

public class OCIMetricKeys {
public static final String LIST = "LIST";
public static final String HEAD = "HEAD";
public static final String WRITE = "WRITE";
public static final String READ = "READ";
public static final String DELETE = "DELETE";
public static final String RENAME = "RENAME";
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package com.oracle.bmc.hdfs.monitoring;

public class OCIMetricWithFBLatency extends OCIMetricWithThroughput {
/**
* The time to first byte when a read operation was performed in milliseconds.
*/
private final double ttfb;
public OCIMetricWithFBLatency(String key, double overallTime, double ttfb, double throughput, Exception e,
double bytesTransferred, String bucketName) {
super(key, overallTime, throughput, e, bytesTransferred, bucketName);
this.ttfb = ttfb;
}

public double getTtfb() {
return ttfb;
}

@Override
public String toString() {
return super.toString() + " OCIMetricWithFBLatency{" +
"ttfb=" + ttfb +
'}';
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package com.oracle.bmc.hdfs.monitoring;

public class OCIMetricWithThroughput extends OCIMetric {
/**
* The throughput that was recorded for the operation in bytes/second
*/
private final double throughput;
/**
* The total count of bytes that were transferred in or out.
*/
private final double bytesTransferred;

public OCIMetricWithThroughput(String key, double overallTime, double throughput, Exception e,
double bytesTransferred, String bucketName) {
super(overallTime, key, e, bucketName);
this.throughput = throughput;
this.bytesTransferred = bytesTransferred;
}

public double getThroughput() {
return throughput;
}

public double getBytesTransferred() { return bytesTransferred; }

@Override
public String toString() {
return super.toString() + " OCIMetricWithThroughput{" +
", throughput=" + throughput +
", bytesTransferred=" + bytesTransferred +
'}';
}
}
Loading

0 comments on commit 83bcd77

Please sign in to comment.