-
Notifications
You must be signed in to change notification settings - Fork 5
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add method to convert a data frame to a JSON string #94
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -7,6 +7,12 @@ | |
.fleet/ | ||
*.iml | ||
|
||
# VS Code specific | ||
.bloop | ||
.metals | ||
metals.sbt | ||
.vscode | ||
|
||
# SBT specific | ||
.bsp/ | ||
coverage.xml | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -155,4 +155,9 @@ class Writeable private[polars] (ptr: Long) { | |
writeMode = _mode | ||
) | ||
} | ||
|
||
def toJsonString(pretty: Boolean, rowOriented: Boolean): String = json(ptr, pretty, rowOriented) | ||
|
||
def toJsonBytes(pretty: Boolean, rowOriented: Boolean): Array[Byte] = jsonBytes(ptr, pretty, rowOriented) | ||
Comment on lines
+159
to
+161
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. three things here,
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. for point 1, another option can be via |
||
|
||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -30,4 +30,8 @@ private[polars] object write extends Natively { | |
writeMode: String | ||
): Unit | ||
|
||
@native def json(ptr: Long, pretty: Boolean, rowOriented: Boolean): String | ||
|
||
@native def jsonBytes(ptr: Long, pretty: Boolean, rowOriented: Boolean): Array[Byte] | ||
|
||
Comment on lines
+33
to
+36
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this needs changes as per the suggestions above. |
||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
package examples.scala.io | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. a separate example might not be needed for every format, but it will be great if this can be converted to a bunch of unit tests instead. current coverage is 0 and is the next thing on my agenda :) |
||
|
||
import org.polars.scala.polars.Polars | ||
import org.polars.scala.polars.api.DataFrame | ||
|
||
import examples.scala.utils.CommonUtils | ||
|
||
/** Polars supports exporting the contents of a [[DataFrame]] to JSON. | ||
* | ||
* It has 2 formats: | ||
* - a row-oriented format, which represents the frame as an array of objects whose keys are | ||
* the column names and whose values are the row’s corresponding values. | ||
* - a column-oriented format, which represents the frame as an array of objects containing a | ||
* column name, type, and the array of column values | ||
* | ||
* The column-oriented format may be pretty-printed. The row-oriented format is less efficient, | ||
* but may be more convenient for downstream applications. | ||
*/ | ||
object Json { | ||
|
||
def main(args: Array[String]) = { | ||
|
||
val path = CommonUtils.getResource("/files/web-ds/data.csv") | ||
val df: DataFrame = Polars.csv.scan(path).collect | ||
|
||
println("Showing CSV file as a DataFrame to stdout.") | ||
df.show() | ||
|
||
println("Showing column-oriented CSV file as a DataFrame to stdout.") | ||
val colOriented = df.write().toJsonString(pretty = false, rowOriented = false) | ||
println(colOriented) | ||
|
||
println("Showing pretty column-oriented CSV file as a DataFrame to stdout.") | ||
val prettyOriented = df.write().toJsonString(pretty = true, rowOriented = false) | ||
println(prettyOriented) | ||
|
||
println("Showing row column-oriented CSV file as a DataFrame to stdout.") | ||
val rowOriented = df.write().toJsonString(pretty = false, rowOriented = true) | ||
println(rowOriented) | ||
|
||
|
||
println("Showing pretty column-oriented CSV file as a DataFrame to stdout.") | ||
val prettyOrientedBytes = df.write().toJsonBytes(pretty = true, rowOriented = false) | ||
println(new String(prettyOrientedBytes, "UTF-8")) | ||
} | ||
|
||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
#![allow(non_snake_case)] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this whole thing needs changes as per the suggestions above. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. you can follow the same file contents as For inspiration you can see write_json and write_ndjson |
||
|
||
use jni::objects::{JObject, JPrimitiveArray}; | ||
use jni::sys::{_jobject, jboolean, jlong}; | ||
use jni::JNIEnv; | ||
use jni_fn::jni_fn; | ||
use polars::prelude::*; | ||
|
||
use crate::j_data_frame::JDataFrame; | ||
|
||
#[jni_fn("org.polars.scala.polars.internal.jni.io.write$")] | ||
pub fn json( | ||
env: JNIEnv, | ||
_object: JObject, | ||
df_ptr: jlong, | ||
pretty: jboolean, | ||
row_oriented: jboolean, | ||
) -> *mut _jobject { | ||
let buf = json_bytes(df_ptr, pretty, row_oriented); | ||
let rust_string = String::from_utf8(buf).unwrap(); | ||
|
||
let output = env | ||
.new_string(rust_string) | ||
.expect("Couldn't create Java string!"); | ||
|
||
output.into_raw() | ||
} | ||
|
||
#[jni_fn("org.polars.scala.polars.internal.jni.io.write$")] | ||
pub fn jsonBytes<'a>( | ||
env: JNIEnv<'a>, | ||
_object: JObject, | ||
df_ptr: jlong, | ||
pretty: jboolean, | ||
row_oriented: jboolean, | ||
) -> JPrimitiveArray<'a, i8> { | ||
let buf = json_bytes(df_ptr, pretty, row_oriented); | ||
env.byte_array_from_slice(&buf).unwrap() | ||
} | ||
|
||
fn json_bytes<'a>(df_ptr: jlong, pretty: jboolean, row_oriented: jboolean) -> Vec<u8> { | ||
let j_df = unsafe { &mut *(df_ptr as *mut JDataFrame) }; | ||
let mut data_frame = j_df.to_owned().df; | ||
|
||
let mut df = data_frame.as_single_chunk_par(); | ||
|
||
let mut buf: Vec<u8> = Vec::new(); | ||
match (pretty == 1, row_oriented == 1) { | ||
(_, true) => JsonWriter::new(&mut buf) | ||
.with_json_format(JsonFormat::Json) | ||
.finish(&mut df), | ||
(true, _) => serde_json::to_writer_pretty(&mut buf, &df) | ||
.map_err(|e| polars_err!(ComputeError: "{e}")), | ||
(false, _) => { | ||
serde_json::to_writer(&mut buf, &df).map_err(|e| polars_err!(ComputeError: "{e}")) | ||
}, | ||
} | ||
.expect("Unable to format JSON"); | ||
|
||
buf | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,4 @@ | ||
pub mod avro; | ||
pub mod ipc; | ||
pub mod json; | ||
pub mod parquet; |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
these can be excluded
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
wanna keep the build light.