{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":89737451,"defaultBranch":"master","name":"spark","ownerLogin":"Affirm","currentUserCanPush":false,"isFork":true,"isEmpty":false,"createdAt":"2017-04-28T19:16:02.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/4098256?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1721691760.0","currentOid":""},"activityList":{"items":[{"before":null,"after":"8f2594f9880db03f4066d3ccd18780fab5b67d9b","ref":"refs/heads/v3.5.1-iceberg","pushedAt":"2024-07-22T23:42:40.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"alanxu00","name":null,"path":"/alanxu00","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/69825788?s=80&v=4"},"commit":{"message":"java 11","shortMessageHtmlLink":"java 11"}},{"before":"ebc45ec16fb50cafb0d0e6a674890c460e52b63c","after":"11a23d39da39ec1a5359b571c4298061be0ecd25","ref":"refs/heads/sundeepk/v3.5.1","pushedAt":"2024-06-07T07:10:29.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"sundeepk-affirm","name":"Sundeep Kambhampati","path":"/sundeepk-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/126815841?s=80&v=4"},"commit":{"message":"hadoop aws 3.3.4","shortMessageHtmlLink":"hadoop aws 3.3.4"}},{"before":"716c3c34bc8dddeb9dd69b737401fe6da5ff71ef","after":"ebc45ec16fb50cafb0d0e6a674890c460e52b63c","ref":"refs/heads/sundeepk/v3.5.1","pushedAt":"2024-06-07T01:49:44.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"sundeepk-affirm","name":"Sundeep Kambhampati","path":"/sundeepk-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/126815841?s=80&v=4"},"commit":{"message":"remove aws sdk","shortMessageHtmlLink":"remove aws sdk"}},{"before":"fd86f85e181fc2dc0f50a096855acf83a6cc5d9c","after":"716c3c34bc8dddeb9dd69b737401fe6da5ff71ef","ref":"refs/heads/sundeepk/v3.5.1","pushedAt":"2024-06-07T01:03:10.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"sundeepk-affirm","name":"Sundeep Kambhampati","path":"/sundeepk-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/126815841?s=80&v=4"},"commit":{"message":"add hadoop-aws","shortMessageHtmlLink":"add hadoop-aws"}},{"before":null,"after":"fd86f85e181fc2dc0f50a096855acf83a6cc5d9c","ref":"refs/heads/sundeepk/v3.5.1","pushedAt":"2024-04-26T17:52:55.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"sundeepk-affirm","name":"Sundeep Kambhampati","path":"/sundeepk-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/126815841?s=80&v=4"},"commit":{"message":"Preparing Spark release v3.5.1-rc2","shortMessageHtmlLink":"Preparing Spark release v3.5.1-rc2"}},{"before":"6d194148f3a46499651bee199ab3d77d68439ede","after":"adce32ef4acc78950d956426df9d46f2457eb991","ref":"refs/heads/affirm-3.3.2","pushedAt":"2024-03-26T21:11:01.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"rayhondo","name":"Ray Kim","path":"/rayhondo","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/8979912?s=80&v=4"},"commit":{"message":"set spark_version env var for pydeequ (#31)","shortMessageHtmlLink":"set spark_version env var for pydeequ (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2209294464\" data-permission-text=\"Title is private\" data-url=\"https://github.com/Affirm/spark/issues/31\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/Affirm/spark/pull/31/hovercard\" href=\"https://github.com/Affirm/spark/pull/31\">#31</a>)"}},{"before":null,"after":"8cb3b1046c7cde1a2cbff00a692eed03fed740a2","ref":"refs/heads/raykim/iceberg/set-spark-version-env-var","pushedAt":"2024-03-26T20:46:33.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"rayhondo","name":"Ray Kim","path":"/rayhondo","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/8979912?s=80&v=4"},"commit":{"message":"set spark_version env var for pydeequ","shortMessageHtmlLink":"set spark_version env var for pydeequ"}},{"before":null,"after":"deb374607167e7554dc260bddbb919d0fbaed65e","ref":"refs/heads/raykim/pyspark-342","pushedAt":"2024-03-12T21:59:57.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"rayhondo","name":"Ray Kim","path":"/rayhondo","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/8979912?s=80&v=4"},"commit":{"message":"fix bug","shortMessageHtmlLink":"fix bug"}},{"before":"4ccd39cd6477fe5bc9bd819284a00e728301a0fd","after":"c405d084d691e0076e8b3ecabd00316db187fbd5","ref":"refs/heads/affirm-3.4.2","pushedAt":"2024-03-12T21:57:00.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"rayhondo","name":"Ray Kim","path":"/rayhondo","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/8979912?s=80&v=4"},"commit":{"message":"Update setup.py","shortMessageHtmlLink":"Update setup.py"}},{"before":"8b4316461e2bc3ca3b72170648ca6b6e36537a65","after":"4ccd39cd6477fe5bc9bd819284a00e728301a0fd","ref":"refs/heads/affirm-3.4.2","pushedAt":"2024-03-12T20:24:42.000Z","pushType":"push","commitsCount":19,"pusher":{"login":"rayhondo","name":"Ray Kim","path":"/rayhondo","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/8979912?s=80&v=4"},"commit":{"message":"upgrade to iceberg 1.5.0 for view support","shortMessageHtmlLink":"upgrade to iceberg 1.5.0 for view support"}},{"before":"0c0e7d4087c64efca259b4fb656b8be643be5686","after":"8b4316461e2bc3ca3b72170648ca6b6e36537a65","ref":"refs/heads/affirm-3.4.2","pushedAt":"2024-03-12T17:55:27.000Z","pushType":"push","commitsCount":57,"pusher":{"login":"rayhondo","name":"Ray Kim","path":"/rayhondo","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/8979912?s=80&v=4"},"commit":{"message":"[SPARK-47305][SQL][TESTS][FOLLOWUP][3.4] Fix the compilation error related to `PropagateEmptyRelationSuite`\n\n### What changes were proposed in this pull request?\nhttps://github.com/apache/spark/pull/45406 has been backported to branch-3.4, where the newly added test case in `PropagateEmptyRelationSuite` uses `DataTypeUtils`, but `DataTypeUtils` is a utility class added in Apache Spark 3.5(SPARK-44475), so this triggered a compilation failure in branch-3.4:\n\n- https://github.com/apache/spark/actions/runs/8183755511/job/22377119069\n\n```\n[error] /home/runner/work/spark/spark/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala:229:27: not found: value DataTypeUtils\n[error]     val schemaForStream = DataTypeUtils.fromAttributes(outputForStream)\n[error]                           ^\n[error] /home/runner/work/spark/spark/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala:233:26: not found: value DataTypeUtils\n[error]     val schemaForBatch = DataTypeUtils.fromAttributes(outputForBatch)\n[error]                          ^\n[info] done compiling\n[info] compiling 1 Scala source to /home/runner/work/spark/spark/connector/connect/common/target/scala-2.12/test-classes ...\n[info] compiling 25 Scala sources and 1 Java source to /home/runner/work/spark/spark/connector/connect/client/jvm/target/scala-2.12/classes ...\n[info] done compiling\n[error] two errors found\n```\n\nTherefore, this PR changes to use the `StructType.fromAttributes` function to fix the compilation failure.\"\n\n### Why are the changes needed?\nFix the compilation failure in branch-3.4\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nPass Github Actions\n\n### Was this patch authored or co-authored using generative AI tooling?\nNo\n\nCloses #45428 from LuciferYang/SPARK-47305-FOLLOWUP-34.\n\nAuthored-by: yangjie01 <yangjie01@baidu.com>\nSigned-off-by: yangjie01 <yangjie01@baidu.com>","shortMessageHtmlLink":"[SPARK-47305][SQL][TESTS][FOLLOWUP][3.4] Fix the compilation error re…"}},{"before":"94689d0f2cbe8e58d5490d392be817bc2b4aec63","after":"9f3cab9f450fcf6b1b4dc8d0ee6df5482175131c","ref":"refs/heads/raykim/iceberg-150","pushedAt":"2024-03-12T17:47:50.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"rayhondo","name":"Ray Kim","path":"/rayhondo","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/8979912?s=80&v=4"},"commit":{"message":"use dev21 for now","shortMessageHtmlLink":"use dev21 for now"}},{"before":null,"after":"94689d0f2cbe8e58d5490d392be817bc2b4aec63","ref":"refs/heads/raykim/iceberg-150","pushedAt":"2024-03-11T17:50:16.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"rayhondo","name":"Ray Kim","path":"/rayhondo","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/8979912?s=80&v=4"},"commit":{"message":"upgrade to iceberg 1.5.0 for view support","shortMessageHtmlLink":"upgrade to iceberg 1.5.0 for view support"}},{"before":"f2f93827edc34ffed4619c7fe88c494e7b537993","after":"2bf8e62bc2fa4acfa3db7cc07ea844051ab44aa0","ref":"refs/heads/affirm-3.3.2-dev4","pushedAt":"2024-02-26T17:12:00.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"roman-affirm","name":"RomanH","path":"/roman-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/108360591?s=80&v=4"},"commit":{"message":"dev version","shortMessageHtmlLink":"dev version"}},{"before":null,"after":"f2f93827edc34ffed4619c7fe88c494e7b537993","ref":"refs/heads/affirm-3.3.2-dev4","pushedAt":"2024-02-26T17:05:29.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"roman-affirm","name":"RomanH","path":"/roman-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/108360591?s=80&v=4"},"commit":{"message":"dev version","shortMessageHtmlLink":"dev version"}},{"before":"ddf611f72c7a9211aba407fb5643c05c5306a922","after":"6d194148f3a46499651bee199ab3d77d68439ede","ref":"refs/heads/affirm-3.3.2","pushedAt":"2024-02-26T10:15:14.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"roman-affirm","name":"RomanH","path":"/roman-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/108360591?s=80&v=4"},"commit":{"message":"[DRPT] Affirm 3.3.2 line sep feature (#28)\n\n* [SPARK-39689][SQL] Support 2-chars `lineSep` in CSV datasource\r\n\r\n### What changes were proposed in this pull request?\r\nUnivocity parser allows to set line separator to 1 to 2 characters ([code](https://github.com/uniVocity/univocity-parsers/blob/master/src/main/java/com/univocity/parsers/common/Format.java#L103)), CSV options should not block this usage ([code](https://github.com/apache/spark/blob/master/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala#L218)). This PR updates the requirement of `lineSep` accepting 1 or 2 characters in `CSVOptions`.\r\n\r\nDue to the limitation of supporting multi-chars `lineSep` within quotes, I propose to leave this feature undocumented and add a WARN message to users.\r\n\r\n### Why are the changes needed?\r\nUnblock the usability of 2 characters `lineSep`.\r\n\r\n### Does this PR introduce _any_ user-facing change?\r\nNo - undocumented feature.\r\n\r\n### How was this patch tested?\r\nNew UT.\r\n\r\nCloses #37107 from Yaohua628/spark-39689.\r\n\r\nLead-authored-by: yaohua <yaohua.zhao@databricks.com>\r\nCo-authored-by: Yaohua Zhao <79476540+Yaohua628@users.noreply.github.com>\r\nSigned-off-by: Hyukjin Kwon <gurwls223@apache.org>\r\n\r\n* bumped version\r\n\r\n---------\r\n\r\nSigned-off-by: Hyukjin Kwon <gurwls223@apache.org>\r\nCo-authored-by: yaohua <yaohua.zhao@databricks.com>\r\nCo-authored-by: Yaohua Zhao <79476540+Yaohua628@users.noreply.github.com>","shortMessageHtmlLink":"[DRPT] Affirm 3.3.2 line sep feature (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2152217950\" data-permission-text=\"Title is private\" data-url=\"https://github.com/Affirm/spark/issues/28\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/Affirm/spark/pull/28/hovercard\" href=\"https://github.com/Affirm/spark/pull/28\">#28</a>)"}},{"before":null,"after":"0c0e7d4087c64efca259b4fb656b8be643be5686","ref":"refs/heads/affirm-3.4.2","pushedAt":"2024-02-24T09:23:09.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"roman-affirm","name":"RomanH","path":"/roman-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/108360591?s=80&v=4"},"commit":{"message":"Preparing Spark release v3.4.2-rc1","shortMessageHtmlLink":"Preparing Spark release v3.4.2-rc1"}},{"before":"3192c8c68423fb2d5f73da238ba60c3d1fb3559f","after":null,"ref":"refs/heads/affirm-3.4.3","pushedAt":"2024-02-24T09:08:12.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"roman-affirm","name":"RomanH","path":"/roman-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/108360591?s=80&v=4"}},{"before":null,"after":"3192c8c68423fb2d5f73da238ba60c3d1fb3559f","ref":"refs/heads/affirm-3.4.3","pushedAt":"2024-02-24T09:07:15.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"roman-affirm","name":"RomanH","path":"/roman-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/108360591?s=80&v=4"},"commit":{"message":"[SPARK-47125][SQL] Return null if Univocity never triggers parsing\n\nThis PR proposes to prevent `null` for `tokenizer.getContext`. This is similar with https://github.com/apache/spark/pull/28029. `getContext` seemingly via the univocity library, it can return null if `begingParsing` is not invoked (https://github.com/uniVocity/univocity-parsers/blob/master/src/main/java/com/univocity/parsers/common/AbstractParser.java#L53). This can happen when `parseLine` is not invoked at https://github.com/apache/spark/blob/e081f06ea401a2b6b8c214a36126583d35eaf55f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala#L300 - `parseLine` invokes `begingParsing`.\n\nTo fix up a bug.\n\nYes. In a very rare case, when `CsvToStructs` is used as a sole predicate against an empty row, it might trigger NPE. This PR fixes it.\n\nManually tested, but test case will be done in a separate PR. We should backport this to all branches.\n\nNo.\n\nCloses #45210 from HyukjinKwon/SPARK-47125.\n\nAuthored-by: Hyukjin Kwon <gurwls223@apache.org>\nSigned-off-by: Hyukjin Kwon <gurwls223@apache.org>\n(cherry picked from commit a87015efb5cf36103bc4eb82ae8613874e2eb408)\nSigned-off-by: Hyukjin Kwon <gurwls223@apache.org>","shortMessageHtmlLink":"[SPARK-47125][SQL] Return null if Univocity never triggers parsing"}},{"before":"3192c8c68423fb2d5f73da238ba60c3d1fb3559f","after":null,"ref":"refs/heads/affirm.3.4.3","pushedAt":"2024-02-24T09:07:04.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"roman-affirm","name":"RomanH","path":"/roman-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/108360591?s=80&v=4"}},{"before":null,"after":"3192c8c68423fb2d5f73da238ba60c3d1fb3559f","ref":"refs/heads/affirm.3.4.3","pushedAt":"2024-02-24T09:06:42.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"roman-affirm","name":"RomanH","path":"/roman-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/108360591?s=80&v=4"},"commit":{"message":"[SPARK-47125][SQL] Return null if Univocity never triggers parsing\n\nThis PR proposes to prevent `null` for `tokenizer.getContext`. This is similar with https://github.com/apache/spark/pull/28029. `getContext` seemingly via the univocity library, it can return null if `begingParsing` is not invoked (https://github.com/uniVocity/univocity-parsers/blob/master/src/main/java/com/univocity/parsers/common/AbstractParser.java#L53). This can happen when `parseLine` is not invoked at https://github.com/apache/spark/blob/e081f06ea401a2b6b8c214a36126583d35eaf55f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala#L300 - `parseLine` invokes `begingParsing`.\n\nTo fix up a bug.\n\nYes. In a very rare case, when `CsvToStructs` is used as a sole predicate against an empty row, it might trigger NPE. This PR fixes it.\n\nManually tested, but test case will be done in a separate PR. We should backport this to all branches.\n\nNo.\n\nCloses #45210 from HyukjinKwon/SPARK-47125.\n\nAuthored-by: Hyukjin Kwon <gurwls223@apache.org>\nSigned-off-by: Hyukjin Kwon <gurwls223@apache.org>\n(cherry picked from commit a87015efb5cf36103bc4eb82ae8613874e2eb408)\nSigned-off-by: Hyukjin Kwon <gurwls223@apache.org>","shortMessageHtmlLink":"[SPARK-47125][SQL] Return null if Univocity never triggers parsing"}},{"before":"3192c8c68423fb2d5f73da238ba60c3d1fb3559f","after":null,"ref":"refs/heads/affirm.3.4","pushedAt":"2024-02-24T09:06:28.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"roman-affirm","name":"RomanH","path":"/roman-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/108360591?s=80&v=4"}},{"before":null,"after":"3192c8c68423fb2d5f73da238ba60c3d1fb3559f","ref":"refs/heads/affirm.3.4","pushedAt":"2024-02-24T09:05:51.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"roman-affirm","name":"RomanH","path":"/roman-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/108360591?s=80&v=4"},"commit":{"message":"[SPARK-47125][SQL] Return null if Univocity never triggers parsing\n\nThis PR proposes to prevent `null` for `tokenizer.getContext`. This is similar with https://github.com/apache/spark/pull/28029. `getContext` seemingly via the univocity library, it can return null if `begingParsing` is not invoked (https://github.com/uniVocity/univocity-parsers/blob/master/src/main/java/com/univocity/parsers/common/AbstractParser.java#L53). This can happen when `parseLine` is not invoked at https://github.com/apache/spark/blob/e081f06ea401a2b6b8c214a36126583d35eaf55f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala#L300 - `parseLine` invokes `begingParsing`.\n\nTo fix up a bug.\n\nYes. In a very rare case, when `CsvToStructs` is used as a sole predicate against an empty row, it might trigger NPE. This PR fixes it.\n\nManually tested, but test case will be done in a separate PR. We should backport this to all branches.\n\nNo.\n\nCloses #45210 from HyukjinKwon/SPARK-47125.\n\nAuthored-by: Hyukjin Kwon <gurwls223@apache.org>\nSigned-off-by: Hyukjin Kwon <gurwls223@apache.org>\n(cherry picked from commit a87015efb5cf36103bc4eb82ae8613874e2eb408)\nSigned-off-by: Hyukjin Kwon <gurwls223@apache.org>","shortMessageHtmlLink":"[SPARK-47125][SQL] Return null if Univocity never triggers parsing"}},{"before":"3192c8c68423fb2d5f73da238ba60c3d1fb3559f","after":null,"ref":"refs/heads/affirm.3.3.4","pushedAt":"2024-02-24T09:05:31.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"roman-affirm","name":"RomanH","path":"/roman-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/108360591?s=80&v=4"}},{"before":null,"after":"3192c8c68423fb2d5f73da238ba60c3d1fb3559f","ref":"refs/heads/affirm.3.3.4","pushedAt":"2024-02-24T09:05:06.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"roman-affirm","name":"RomanH","path":"/roman-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/108360591?s=80&v=4"},"commit":{"message":"[SPARK-47125][SQL] Return null if Univocity never triggers parsing\n\nThis PR proposes to prevent `null` for `tokenizer.getContext`. This is similar with https://github.com/apache/spark/pull/28029. `getContext` seemingly via the univocity library, it can return null if `begingParsing` is not invoked (https://github.com/uniVocity/univocity-parsers/blob/master/src/main/java/com/univocity/parsers/common/AbstractParser.java#L53). This can happen when `parseLine` is not invoked at https://github.com/apache/spark/blob/e081f06ea401a2b6b8c214a36126583d35eaf55f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala#L300 - `parseLine` invokes `begingParsing`.\n\nTo fix up a bug.\n\nYes. In a very rare case, when `CsvToStructs` is used as a sole predicate against an empty row, it might trigger NPE. This PR fixes it.\n\nManually tested, but test case will be done in a separate PR. We should backport this to all branches.\n\nNo.\n\nCloses #45210 from HyukjinKwon/SPARK-47125.\n\nAuthored-by: Hyukjin Kwon <gurwls223@apache.org>\nSigned-off-by: Hyukjin Kwon <gurwls223@apache.org>\n(cherry picked from commit a87015efb5cf36103bc4eb82ae8613874e2eb408)\nSigned-off-by: Hyukjin Kwon <gurwls223@apache.org>","shortMessageHtmlLink":"[SPARK-47125][SQL] Return null if Univocity never triggers parsing"}},{"before":"3f05528a51fded3282b896290dc64fe71b35c8cb","after":"f9df296282bd9c9d1831c59ee335f6c232079db1","ref":"refs/heads/affirm-3.3.2-line-sep-feature","pushedAt":"2024-02-24T08:53:05.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"roman-affirm","name":"RomanH","path":"/roman-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/108360591?s=80&v=4"},"commit":{"message":"bumped version","shortMessageHtmlLink":"bumped version"}},{"before":null,"after":"3f05528a51fded3282b896290dc64fe71b35c8cb","ref":"refs/heads/affirm-3.3.2-line-sep-feature","pushedAt":"2024-02-24T08:49:16.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"roman-affirm","name":"RomanH","path":"/roman-affirm","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/108360591?s=80&v=4"},"commit":{"message":"[SPARK-39689][SQL] Support 2-chars `lineSep` in CSV datasource\n\n### What changes were proposed in this pull request?\nUnivocity parser allows to set line separator to 1 to 2 characters ([code](https://github.com/uniVocity/univocity-parsers/blob/master/src/main/java/com/univocity/parsers/common/Format.java#L103)), CSV options should not block this usage ([code](https://github.com/apache/spark/blob/master/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala#L218)). This PR updates the requirement of `lineSep` accepting 1 or 2 characters in `CSVOptions`.\n\nDue to the limitation of supporting multi-chars `lineSep` within quotes, I propose to leave this feature undocumented and add a WARN message to users.\n\n### Why are the changes needed?\nUnblock the usability of 2 characters `lineSep`.\n\n### Does this PR introduce _any_ user-facing change?\nNo - undocumented feature.\n\n### How was this patch tested?\nNew UT.\n\nCloses #37107 from Yaohua628/spark-39689.\n\nLead-authored-by: yaohua <yaohua.zhao@databricks.com>\nCo-authored-by: Yaohua Zhao <79476540+Yaohua628@users.noreply.github.com>\nSigned-off-by: Hyukjin Kwon <gurwls223@apache.org>","shortMessageHtmlLink":"[SPARK-39689][SQL] Support 2-chars <code>lineSep</code> in CSV datasource"}},{"before":"ebd52e785845bd727665d2a57947918f2691cd08","after":"ddf611f72c7a9211aba407fb5643c05c5306a922","ref":"refs/heads/affirm-3.3.2","pushedAt":"2023-11-24T22:07:19.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"rayhondo","name":"Ray Kim","path":"/rayhondo","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/8979912?s=80&v=4"},"commit":{"message":"patch iceberg delete bug fix (#27)","shortMessageHtmlLink":"patch iceberg delete bug fix (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2010243484\" data-permission-text=\"Title is private\" data-url=\"https://github.com/Affirm/spark/issues/27\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/Affirm/spark/pull/27/hovercard\" href=\"https://github.com/Affirm/spark/pull/27\">#27</a>)"}},{"before":null,"after":"98e52d1c12229697b3af9de903348f1cf489ea1f","ref":"refs/heads/raykim/iceberg/patch-delete-fix","pushedAt":"2023-11-24T21:47:18.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"rayhondo","name":"Ray Kim","path":"/rayhondo","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/8979912?s=80&v=4"},"commit":{"message":"patch iceberg delete bug fix","shortMessageHtmlLink":"patch iceberg delete bug fix"}},{"before":"4ff19e7e6da9fce9de3c8d52ff29ac0489b4abd9","after":"ebd52e785845bd727665d2a57947918f2691cd08","ref":"refs/heads/affirm-3.3.2","pushedAt":"2023-11-20T20:31:04.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"rayhondo","name":"Ray Kim","path":"/rayhondo","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/8979912?s=80&v=4"},"commit":{"message":"[iceberg] upgrade iceberg spark runtime to 1.4.2 (#26)\n\n* upgrade iceberg spark runtime to 1.4.2\r\n\r\n* add iceberg-aws-bundle + version bump\r\n\r\n* add back mysql at top of pom.xml\r\n\r\n* remove redundant mysql connector\r\n\r\n* use non-dev version","shortMessageHtmlLink":"[iceberg] upgrade iceberg spark runtime to 1.4.2 (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"1999644277\" data-permission-text=\"Title is private\" data-url=\"https://github.com/Affirm/spark/issues/26\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/Affirm/spark/pull/26/hovercard\" href=\"https://github.com/Affirm/spark/pull/26\">#26</a>)"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"Y3Vyc29yOnYyOpK7MjAyNC0wNy0yMlQyMzo0Mjo0MC4wMDAwMDBazwAAAASGSjit","startCursor":"Y3Vyc29yOnYyOpK7MjAyNC0wNy0yMlQyMzo0Mjo0MC4wMDAwMDBazwAAAASGSjit","endCursor":"Y3Vyc29yOnYyOpK7MjAyMy0xMS0yMFQyMDozMTowNC4wMDAwMDBazwAAAAOzqAR6"}},"title":"Activity · Affirm/spark"}