Selection of useful reusable components
By default, Scala 2.11 is used. To build Commons for another Scala version, switch to the required Scala version first.
# E.g. to switch to Scala 2.13 use
mvn scala-cross-build:change-version -Pscala-2.13
When building the project activate a Scala profile corresponding to the Scala version of the codebase.
# E.g. for Scala 2.13 use
mvn clean install -Pscala-2.13
./build-all.sh
./mvn clean verify -Pcode-coverage
Code coverage will be generated on path:
{local-path}\commons\target\jacoco
import za.co.absa.commons.lang.extensions.AnyExtension._
// Optionally call a method in a chain
new MyBuilder
.withX(42)
.withY(77)
.optionally(_.withZ, maybeZ) // <---- withZ is called with a `value` if `maybeZ` is `Some(value)`
.optionally(_.withABC(a, _, c), maybeB) // <---- it also works with n-ary methods
// Alternatively `having()` method can be used. It does the same thing as `optionally()`,
// but because of re-arranged method parameters it's easier for the compiler to infer types.
// See: https://github.com/AbsaOSS/commons/issues/56
new MyBuilder
.withX(42)
.withY(77)
.having(maybeZ)(_.withZ)
// Conditionally call a method in a chain
new MyBuilder
.withX(42)
.withY(77)
.when(shouldIncludeZ)(_.withZ(88))
// ...same as `when()` but with inverted condition
new MyBuilder
.withX(42)
.withY(77)
.unless(shouldExcludeZ)(_.withZ(88))
import za.co.absa.commons.lang.extensions.ArrayExtension._
val arr = Array(1, 2, 3)
// removes duplicates in an array;
// an argument to distinctBy is a function which projects each array value into another value
// that is used to determine whether two elements are duplicated
val duplicatesEliminated = arr.distinctBy(identity)
import za.co.absa.commons.lang.extensions.IteratorExtension._
val iter: Iterator[_] = ???
val arr: Array[_] = ???
// copy 42 items from the `iter` to the `arr` with array offset 7
iter.fetchToArray(arr, 7, 42) // returns a number of actually copied items
import za.co.absa.commons.lang.extensions.NonOptionExtension._
// returns the object as Some(_) if anyNonOptionObject is not null, None otherwise
anyNonOptionObject.toOption
import za.co.absa.commons.lang.extensions.OptionExtension._
val someOption = Some("abc")
// returns Success("abc")
someOption.toTry(new Exception)
val noneOption = None
val e = new Exception
// returns Failure(e)
someOption.toTry(e)
import za.co.absa.commons.lang.extensions.SeqExtension._
Seq(1, 2, 2, 2, 1).groupConsecutiveBy[Int](a => a) // Seq(Seq(1), Seq(2, 2, 2), Seq(1))
Seq(1, 2, 2, 2, 1).groupConsecutiveBy[Int](a => 1) // Seq(Seq(1, 2, 2, 2, 1))
Seq(1, 24, 27, 2, 1).groupConsecutiveBy[Int](a => a.toString.length) // Seq(Seq(1), Seq(24, 27), Seq(2, 1))
Seq(1, 2, 2, 2, 1).groupConsecutiveByPredicate(a => a == 2) // Seq(Seq(1), Seq(2, 2, 2), Seq(1))
Seq(1, 1, 2, 2, 4, 1).groupConsecutiveByPredicate(a => a == 2) // Seq(Seq(1, 1), Seq(2), Seq(2), Seq(4), Seq(1))
Seq(1, 24, 27, 2, 1).groupConsecutiveByOption[Int](
a => if(a.toString.length > 1) Some(a.toString.length) else None
) // Seq(Seq(1), Seq(24, 27), Seq(2), Seq(1))
import za.co.absa.commons.lang.extensions.StringExtension._
"abcba".replaceChars(Map('a' -> 'b', 'b' -> 'a')) // "bacab"
"Hello world".findFirstUnquoted(Set('w', 'e', 'l'), Set.empty) // Some(1)
"Hello world".findFirstUnquoted(Set('w'), Set.empty) // Some(6)
"Hello world".findFirstUnquoted(Set('a'), Set.empty) // None
"Hello `w`orld".findFirstUnquoted(Set('w'), '`') // None
"Hello `world".findFirstUnquoted(Set('w'), '`') // Some(7)
"`Hello` \\'world".findFirstUnquoted(Set('w', 'e', 'l'), Set('\'', '`')) // Some(10)
"Hello world".hasUnquoted(Set('w', 'e', 'l'), Set('`')) // true
"`Hello world`".hasUnquoted(Set('w', 'e', 'l'), Set('`')) // false
"Lorem i ipsum".countUnquoted(Set('o', 'i'), Set.empty) // Map('o' -> 1, 'i' -> 2)
"Lorem `i` ipsum".countUnquoted(Set('o', 'i'), Set('`')) // Map('o' -> 1, 'i' -> 1)
"aaa" / "123" // "aaa/123"
"aaa/" / "123" // "aaa/123"
"a".nonEmptyOrElse("b") // "a"
"".nonEmptyOrElse("b") // "b"
"".coalesce("A", "") // "A"
"".coalesce("", "", "B", "", "C") // "B"
"X".coalesce("Y", "Z") // "X"
(null: String).nonBlankOption // None
" ".nonBlankOption // None
" foo bar 42 ".nonBlankOption // Some(" foo bar 42 ")
import za.co.absa.commons.lang.extensions.TraversableExtension._
Traversable(1, 2, 3).toNonEmptyOption // Some(Traversable(1, 2, 3))
Traversable().toNonEmptyOption // None
import za.co.absa.commons.lang.extensions.TraversableOnceExtension._
List(1, 2).distinctBy(identity) // List(1, 2)
List(1, 2, 1).distinctBy(identity) // List(1, 2)
List(1, 2, 1, 0, 5).distinctBy(a => a % 2) // List(1, 2)
Warning: these are deprecated.
Use type-specific ...Extension
instead, for example, za.co.absa.commons.lang.extensions.IteratorExtension
.
import CollectionImplicits._
val iter: Iterator[_] = ???
val arr: Array[_] = ???
// copy 42 items from the `iter` to the `arr` with array offset 7
iter.fetchToArray(arr, 7, 42) // returns a number of actually copied items
import CollectionImplicits._
// Get distinct elements by only comparing certain property(-es)
val xs = Seq(
Foo(x = 1, ...), // A
Foo(x = 2, ...), // B
Foo(x = 1, ...), // C
Foo(x = 2, ...), // D
Foo(x = 3, ...), // E
)
xs.distinctBy(_.x) // returns elements A, B, E
val myNodes: Seq[MyNode] = ??? // an arbitrary sequence of objects that can represent graph nodes
// import extension methods
import za.co.absa.commons.graph.GraphImplicits._
val sortedNodes = myNodes.sortedTopologicallyBy(_.id, _.refIds) // arguments are functions that return a self ID and outbound IDs for every node in the collection
// ... or using implicit `DAGNodeIDMapping` instance instead of explicitly passing mapping functions as arguments
implicit object MyNodeIdMapping extends DAGNodeIdMapping[MyNode, NodeId] {
override def selfId(n: MyNode): NodeId = ???
override def refIds(n: MyNode): Traversable[NodeId] = ???
}
val sortedNodes = myNodes.sortedTopologically()
A simple stackable Converter
trait with a simple memoized wrapper.
// 1. Define your converter
class AlchemicalConverter extends Converter {
override type From = Lead
override type To = Gold
override def convert(arg: Lead): Gold = ??? // treat with dragon eyes and cosmic rays
}
// 2. Instantiate it
val forge = new AlchemicalConverter
// or a memoized variant
val forge = new AlchemicalConverter with CachingConverter
// 3. Use it
forge.convert(... some lead ...) // returns some gold
Utility object that defines extended type constraints to be used in Scala type definitions.
In particular it defines a not
type constraint.
trait VegetarianMenu {
def add[A <: Food : not[Meat]#λ](food: A)
}
Warning: these are deprecated.
Use type-specific ...Extension
instead, for example, za.co.absa.commons.lang.extensions.StringExtension
.
// Strings
(null: String).nonBlankOption // == None
" \t \n \r ".nonBlankOption // == None
"foo bar".nonBlankOption // == Some("foo bar")
// Collections
Seq.empty.asOption // == None
Seq(1, 2).asOption // == Some(Seq(1, 2))
// Just another way of doing Option(foo)
foo.asOption
// Optionally call a method in a chain
new MyBuilder
.withX(42)
.withY(77)
.optionally(_.withZ, maybeZ) // <---- withZ is called with a `value` if `maybeZ` is `Some(value)`
.optionally(_.withABC(a, _, c), maybeB) // <---- it also works with n-ary methods
// Alternatively `having()` method can be used. It does the same thing as `optionally()`,
// but because of re-arranged method parameters it's easier for the compiler to infer types.
// See: https://github.com/AbsaOSS/commons/issues/56
new MyBuilder
.withX(42)
.withY(77)
.having(maybeZ)(_.withZ)
...
Parses a connection string containing one or multiple URIs into a list of strings (each being one URI). Input URIs are supposed to have semi-colon-separated base URIs, and each can have multiple comma-separated hosts.
val connectionString = "https://localhost:8080,host2:8080/rest_api;http://localhost:9000/rest_api"
UrisConnectionStringParser.parse(connectionString)
// List("https://localhost:8080/rest_api", "https://host2:8080/rest_api", "http://localhost:9000/rest_api")
Useful methods for org.apache.commons.configuration.Configuration
.
import za.co.absa.commons.config.ConfigurationImplicits._
// return value or throw
conf.getRequiredInt("property.key")
// return Some(value) or None
conf.getOptionalInt("property.key")
// return Map("conf.key1" -> 123, "conf.key2" -> 456)
conf.toMap[Int]
Available for String, Array[String], Boolean, BigDecimal, Byte, Short, Int, Float, Long and Double.
This is an extension of EnvironmentConfiguration
that converts key names between
dot-separated camelCase notation and UPPER_SNAKE_CASE notation which is common for naming environment variables.
See: #54
// Any of the following calls returns a value of FOO_BAR_BAZ environment variable
(new UpperSnakeCaseEnvironmentConfiguration).getString("foo.bar.baz")
(new UpperSnakeCaseEnvironmentConfiguration).getString("fooBarBaz")
(new UpperSnakeCaseEnvironmentConfiguration).getString("foo.barBaz")
Trait ConfTyped
provides a DSL for creating a typed hierarchical configuration object.
It provides access to two main abstractions: Conf
and Prop
Example:
import za.co.absa.commons.config._
object MyAppConfig extends ConfTyped {
val confSource: java.util.Properties = ???
override val rootPrefix = "com.example"
object Foo extends Conf("foo") {
object Bar extends Conf("bar") {
val baz: String = confSource getProperty Prop("baz")
val qux: String = confSource getProperty Prop("qux")
}
}
}
// somewhere in your application
import MyAppConfig._
val baz = Foo.Bar.baz // mapped to the key "com.example.foo.bar.baz" in the <code>confSource</code>
val baz = Foo.Bar.qux // mapped to the key "com.example.foo.bar.qux" in the <code>confSource</code>
Note that ConfTyped
doesn't impose or depend on the way how the configuration values are loaded.
It only provides a convenient way to implicitly construct the configuration key names from the nested object structure.
The key names are obtained by calling Prop("...")
method.
It returns a full property key name that reflects the nesting structure of the Conf
instances' names,
concatenated with dot (.
) and prefixed with the rootPrefix
if one is provided.
Another example of usage ConfTyped
:
val props = new java.util.Properties with ConfTyped {
val foo = new Conf("foo") {
val bar = new Conf("bar") {
lazy val baz = getProperty(Prop("baz"))
}
}
}
props.put("foo.bar.baz", "42")
println(props.foo.bar.baz) // prints 42
ReflectionUtils.directSubClassesOf[Food] // == Seq(classOf[Vegetables], classOf[Meat], classOf[Fish])
ReflectionUtils.objectsOf[Currency] // == Seq(classOf[EUR], classOf[USD], classOf[CZK])
ReflectionUtils.objectForName[MySingleton]("com.example.MySingleton") // == MySingleton
objectForName
with more descriptive exception message in case there is something wrong with provided name.
ReflectionUtils.objectForNameWithDescriptiveException[MySingleton]("com.example.MySingleton") // == MySingleton
ReflectionUtils.extractFieldValue[Int](foo, "bar")
// or if you know a type where the field is declared
ReflectionUtils.extractFieldValue[Doh, Int](foo, "bar")
case class Person(name: String, age: Int, sex: Sex)
val aPerson = Person("Alex", 41, Male)
ReflectionUtils.extractProperties(aPerson) // == Map("name" -> "Alex, "age" -> 42, "sex" -> Male)
case class Button(title: String, isPressed = false)
ReflectionUtils.caseClassCtorArgDefaultValue[Int](classOf[Button], "name") // == None
ReflectionUtils.caseClassCtorArgDefaultValue[Int](classOf[Button], "isPressed") // == Some(false)
ReflectionUtils.ReflectionUtils.allInterfacesOf[A]
// or
ReflectionUtils.ReflectionUtils.allInterfacesOf(aClass)
Can be used to e.g. in a Case Object Enumeration pattern.
A similar solution and the motivation is well describes in the
Scala Enumerations hell article.
But unlike the above approach EnumerationMacros.sealedInstancesOf
utilizes Scala compiler macros,
so that the instances are lookup at the compile time.
sealed trait Color
object Color {
// returns Set(Red, Green, Blue)
val values: Set[Color] = EnumerationMacros.sealedInstancesOf[Color]
case object Red extends Color
case object Green extends Color
case object Blue extends Color
}
If you wants some code to be linked and executed at run-time, here's a simple way to do it:
val fn = ReflectionUtils.compile(q""" some scala code """)
fn()
Of with input parameters:
val fn = ReflectionUtils.compile(q"""
val foo = arg("foo")
val bar = arg("bar")
foo + bar
""")
fn(Map("foo" -> ..., "bar" -> ...))
Sometimes you need to support different versions of some library with a breaking changes in API. If there aren't too many breaking changes, or you only use a certain subset of an API then creating a proper adapter layer could be an overkill. In that case simple run-time evaluation could be a decent alternative:
object FilenameExtractor extends AccessorMethodValueExtractor[String]("filename", "name", "file")
val FilenameExtractor(fileName) = someObjectRepresentingAFile
// The first matching accessor name with type wins
If you try to pattern-match on a type that is missing from the classpath at runtime (e.g. optional dependency) you'll get NoClassDefFoundError
. Though it looks strange as from the use case perspective if there is no Foo
class there couldn't be a Foo
instance. Logically one would expect it to just not match, but in fact it throws.
aObject match {
...
case foo: Foo => // <----- this could throw NoClassDefFoundError !!
...
}
To get a desired behavior you can use SafeTypeMatchingExtractor
:
object FooExtractor extends SafeTypeMatchingExtractor("com.example.Foo")
aObject {
...
case FooExtractor(foo) => // do something with `foo`
...
}
or you can even make a fancy DSL from it:
object `_: Foo` extends SafeTypeMatchingExtractor("com.example.Foo")
aObject {
...
case `_: Foo`(foo) => // do something with `foo`
...
}
A singleton that parses build.properties
from the classpath and return version and timestamp as constants.
Copy build.properties.template
file and paste in into your classpath root as build.properties
. Make sure the resource filtering is enabled on your project build.
Then you can access it's content as simply as this:
BuildInfo.Version // returns `build.version` property from the `build.properties` file
BuildInfo.Timestamp // returns `build.timestamp` property from the `build.properties` file
BuildInfo.BuildProps // returns entire `build.properties` content as immutable Java `Properties`
If needed, you can customize a build.properties
resource path and/or the property mapping.
// loads '/foo/bar.properties' from the classpath
object MyBuildInfo extends BuildInfo(resourcePrefix = "/foo/bar")
object MyBuildInfo extends BuildInfo(propMapping = PropMapping(
version = "bld.ver", // binds "Version" field to "bld.ver" property
timestamp = "bld.ttt" // binds "Timestamp" field to "bld.ttt" property
))
You can also use apply()
method instead of inheritance. It all depends on your preferred code style:
val myBuildInfo = BuildInfo(...)
In client-server application the errors sent to a client is often sanitised for privacy and security reasons. This however complicates troubleshooting because it's difficult to find a much between a client error message and the corresponding exception details in the server logs.
One way to solve this issue is to generate a unique identifier that is then incorporated into the server log on one hand, and is sent to the client along with a client friendly error message on the other hand. Such unique ID will be easy to lookup in logs, and will precisely identify the root cause of the error seen by the client.
// somewhere on the server
try {
service.doSomething()
} catch {
case NonFatal(e) =>
import za.co.absa.commons.error
val errorRef = ErrorRef(e, "oops!")
clientResponse.sendError(errorRef)
}
This way the exception e
is silently logged into the server logs with the message
[ERROR] ... ERROR_ID [123e4567-e89b-12d3-a456-426614174000] oops!
caused by: NullPointerException in ...
<stack trace>
... while the client receives a serialized representation that only contains the error UUID, timestamp and the message "oops!".
For example:
{
"errorId": "123e4567-e89b-12d3-a456-426614174000",
"timestamp": 1611945666787,
"message": "oops!"
}
An easy way to create a temporary file or directory with the support for automatic recursive deletion (as rm -rf
) on JVM shutdown.
val myTmpFile: Path = TempFile.deleteOnExit.path
val myTmpDir: Path = TempDirectory.deleteOnExit.path
val myTmpFile: String = TempFile.deleteOnExit.asString
val myTmpDir: String = TempDirectory.deleteOnExit.asSTring
val myTmpFile: URI = TempFile.deleteOnExit.toURI
val myTmpDir: URI = TempDirectory.deleteOnExit.toURI
It also mimics Java IO API for a similar purpose
TempFile("myPrefix", "mySuffix")
An object containing useful functions that operate on local file system.
val doesExist = LocalFileSystemUtils.localExists("/user/u1/somefile") // true if this file exists, false otherwise
if(doesExist) {
val fileContent = LocalFileSystemUtils.readLocalFile("/user/u1/somefile") // full file as string
}
val tildeReplaced = LocalFileSystemUtils.replaceHome("~/Projects/somedir") // path with replaces tilde with home directory path
A set of stackable traits, serving a wrapper around the way how Json4s (de)serializers are created. Instead of relying on implicit Formats
objects a stackable traits are used.
This API is also binary compatible to Json4s 3.2 and 3.3+ versions (Jackson and Native)
There are two default SerDe implementation that you can use out of the box:
DefaultJacksonJsonSerDe
DefaultNativeJsonSerDe
class MyApp extends App with DefaultJacksonJsonSerDe {
FooBar.toJson // returns JSON string
FooBar.toPrettyJson // returns formatted JSON string
"{...}".fromJson[FooBar] // returns a FooBar instance
}
Or you can create a singleton and use that instead:
object JsonSerDe extends DefaultNativeJsonSerDe
import JsonSerDe._
fooBar.toJson
If you want another parser impl, then you do this:
object MyJsonSerDe
extends AbstractJsonSerDe[MyJson]
with my.JsonMethods
with DefaultFormatsBuilder
import MyJsonSerDe._
fooBar.toJson
If you want custom formats then instead of mixing in DefaultFormatsBuilder
simply override def formats
method.
A simple utility that parses version strings. It supports SemVer 2.0 as well as a simple dot-separated version format. Can be used to compare the versions, for instance when implementing version predicates.
import Version._
require(Version.asSimple(SPARK_VERSION) > ver"2.4")
// or
require(Version.asSemVer(SomeLibVersion) > semver"1.2.3-beta.2")
To get a string representation from a Version
instance asString
extension method can be used:
val myVer: Version = semver"1.2.3-beta.2+777.42"
myVer.asString // returns "1.2.3-beta.2+777.42"
Semantic Versioning specific operations:
import Version._
val myVer = semver"111.222.333-alpha.444+build.555"
myVer.major == 111
myVer.minor == 222
myVer.patch == 333
myVer.core == semver"111.222.333"
myVer.preRelease == ver"alpha.444"
myVer.buildMeta == ver"build.555"
Provides simple means of checking a string to appear to be a valid S3 Location and parsing it into a S3Location
.
That way, one can easily obtain the protocol
, bucketName
, and path
.
- recognized
protocol
s ares3
,s3n
ands3a
bucketName
is checked according to the official S3 Bucket naming rules and official S3 Access Point Alias naming rulespath
content is not checked in any way
import za.co.absa.commons.s3._
import za.co.absa.commons.s3.SimpleS3Location._
"s3a://mybucket.some.where/my/path1".isValidS3Path // yields true
val s3loc: S3Location = "s3://mybucket-123/path/to/file.ext".toSimpleS3Location.get
s3loc.protocol // holds "s3"
s3loc.bucketName // holds "mybucket-123"
s3loc.path // holds "path/to/file.ext"
"s3x://bogus#$%/xxx".toSimpleS3Location // yields None
"s3a://mybucket.some/my/path1".withTrailSlash // yields SimpleS3Location("s3a", "mybucket.some", "my/path1/")
"s3a://mybucket.some/my/path1/".withoutTrailSlash // yields SimpleS3Location("s3a", "mybucket.some", "my/path1")
Copyright 2019 ABSA Group Limited
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.