Skip to content

Commit

Permalink
Update hadoop-aws, hadoop-client to 3.4.0 (#3457)
Browse files Browse the repository at this point in the history
## About this PR
📦 Updates 
* org.apache.hadoop:hadoop-aws
* org.apache.hadoop:hadoop-client

 from `3.3.6` to `3.4.0`

## Usage
✅ **Please merge!**

I'll automatically update this PR to resolve conflicts as long as you
don't change it yourself.

If you'd like to skip this version, you can just close this PR. If you
have any feedback, just mention me in the comments below.

Configure Scala Steward for your repository with a
[`.scala-steward.conf`](https://github.com/scala-steward-org/scala-steward/blob/7ec418cd5441d449f037bca6d34326534c07a5dd/docs/repo-specific-configuration.md)
file.

_Have a fantastic day writing Scala!_

<details>
<summary>⚙ Adjust future updates</summary>

Add this to your `.scala-steward.conf` file to ignore future updates of
this dependency:
```
updates.ignore = [ { groupId = "org.apache.hadoop" } ]
```
Or, add this to slow down future updates of this dependency:
```
dependencyOverrides = [{
  pullRequests = { frequency = "30 days" },
  dependency = { groupId = "org.apache.hadoop" }
}]
```
</details>

<sup>
labels: library-update
</sup>
  • Loading branch information
xerial-bot authored Mar 21, 2024
1 parent ae35820 commit 383ffd4
Show file tree
Hide file tree
Showing 3 changed files with 5 additions and 29 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
*/
package wvlet.airframe.parquet

import com.amazonaws.auth.profile.ProfileCredentialsProvider
import org.apache.hadoop.conf.Configuration
import org.apache.parquet.filter2.compat.FilterCompat
import org.apache.parquet.filter2.predicate.FilterApi
Expand Down Expand Up @@ -178,27 +177,4 @@ object ParquetTest extends AirSpec {
}
}
}

test("write to S3") {
skip("Requires a real S3 bucket to test this code")

val conf = new Configuration()
conf.set("fs.s3a.aws.credentials.provider", "wvlet.airframe.parquet.CustomCredentialProvider")
val path = "s3a://leo-prizm-dev/test.parquet"
withResource(
Parquet.newWriter[SampleRecord](path = path, hadoopConf = conf)
) { writer =>
writer.write(r1)
writer.write(r2)
}

withResource(Parquet.newReader[SampleRecord](path = path, hadoopConf = conf)) { reader =>
reader.read() shouldBe r1
reader.read() shouldBe r2
reader.read() shouldBe null
}
Parquet.readSchema(path, conf)
}
}

class CustomCredentialProvider extends ProfileCredentialsProvider("engineering")
4 changes: 2 additions & 2 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -920,9 +920,9 @@ lazy val parquet =
description := "Parquet columnar format reader/writer support",
libraryDependencies ++= Seq(
"org.apache.parquet" % "parquet-hadoop" % PARQUET_VERSION,
"org.apache.hadoop" % "hadoop-client" % "3.3.6" % Provided,
"org.apache.hadoop" % "hadoop-client" % "3.4.0" % Provided,
// For S3 support
"org.apache.hadoop" % "hadoop-aws" % "3.3.6" % Provided,
"org.apache.hadoop" % "hadoop-aws" % "3.4.0" % Provided,
"software.amazon.awssdk" % "auth" % "2.25.1" % Provided,
// For Apple Silicon (M1)
"org.xerial.snappy" % "snappy-java" % "1.1.10.5",
Expand Down
6 changes: 3 additions & 3 deletions docs/airframe-parquet.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@ airframe-parquet is a library for reading and writing for Scala objects using Pa
libraryDependencies ++= Seq(
"org.wvlet.airframe" %% "airframe-parquet" % "(version)"
// Use your own hadoop version
"org.apache.hadoop" % "hadoop-client" % "3.3.1",
"org.apache.hadoop" % "hadoop-client" % "3.4.0",
// [Optional] For supporting S3
"org.apache.hadoop" % "hadoop-aws" % "3.3.1",
"org.apache.hadoop" % "hadoop-aws" % "3.4.0",
// [Optional] For using custom AWS credential provider
"software.amazon.awssdk" % "auth" % "2.17.18"
"software.amazon.awssdk" % "auth" % "2.25.13"
)
```

Expand Down

0 comments on commit 383ffd4

Please sign in to comment.