Skip to content

Commit ba062c5

Browse files
regadasregadas
committed
Add gcs support
1 parent 73b3ef2 commit ba062c5

File tree

5 files changed

+287
-54
lines changed

5 files changed

+287
-54
lines changed

‎build.sbt

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ lazy val core = project
5151
.settings(
5252
name := "tfr-core",
5353
libraryDependencies ++= Seq(
54+
gcs,
5455
catsCore,
5556
fs2Io,
5657
guava,
@@ -75,9 +76,11 @@ lazy val cli = project
7576
name in GraalVMNativeImage := "tfr",
7677
graalVMNativeImageOptions ++= Seq(
7778
"-H:+ReportExceptionStackTraces",
79+
"-H:EnableURLProtocols=http,https",
7880
"-H:ReflectionConfigurationFiles=" + baseDirectory.value / "src" / "graal" / "reflect-config.json",
7981
"--no-fallback",
80-
"--initialize-at-build-time"
82+
"--initialize-at-build-time",
83+
"--allow-incomplete-classpath"
8184
)
8285
)
8386
.dependsOn(core)

‎cli/src/graal/reflect-config.json

Lines changed: 231 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,232 @@
11
[
2-
{
3-
"name":"org.tensorflow.example.Example",
4-
"allDeclaredMethods":true
5-
},
6-
{
7-
"name":"org.tensorflow.example.Feature",
8-
"allDeclaredMethods":true
9-
},
10-
{
11-
"name":"org.tensorflow.example.Feature$Builder",
12-
"allDeclaredMethods":true
13-
},
14-
{
15-
"name":"org.tensorflow.example.Features",
16-
"allDeclaredMethods":true
17-
},
18-
{
19-
"name":"org.tensorflow.example.Example$Builder",
20-
"allDeclaredMethods":true
21-
},
22-
{
23-
"name":"org.tensorflow.example.BytesList",
24-
"allDeclaredMethods":true
25-
},
26-
{
27-
"name":"org.tensorflow.example.BytesList$Builder",
28-
"allDeclaredMethods":true
29-
},
30-
{
31-
"name":"org.tensorflow.example.FloatList",
32-
"allDeclaredMethods":true
33-
},
34-
{
35-
"name":"org.tensorflow.example.FloatList$Builder",
36-
"allDeclaredMethods":true
37-
},
38-
{
39-
"name":"org.tensorflow.example.Int64List",
40-
"allDeclaredMethods":true
41-
},
42-
{
43-
"name":"org.tensorflow.example.Int64List$Builder",
44-
"allDeclaredMethods":true
45-
}
46-
47-
]
2+
{
3+
"name":"com.google.api.client.googleapis.services.AbstractGoogleClientRequest",
4+
"allDeclaredFields":true
5+
},
6+
{
7+
"name":"com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest",
8+
"allDeclaredFields":true
9+
},
10+
{
11+
"name":"com.google.api.client.http.GenericUrl",
12+
"allDeclaredFields":true
13+
},
14+
{
15+
"name":"com.google.api.client.http.HttpHeaders",
16+
"allDeclaredFields":true,
17+
"allDeclaredMethods":true
18+
},
19+
{
20+
"name":"com.google.api.client.json.GenericJson",
21+
"allDeclaredFields":true,
22+
"methods":[{"name":"<init>","parameterTypes":[] }]
23+
},
24+
{
25+
"name":"com.google.api.client.util.GenericData",
26+
"allDeclaredFields":true,
27+
"methods":[{"name":"<init>","parameterTypes":[] }]
28+
},
29+
{
30+
"name":"com.google.api.services.storage.Storage$Objects$Get",
31+
"allDeclaredFields":true,
32+
"allDeclaredMethods":true
33+
},
34+
{
35+
"name":"com.google.api.services.storage.StorageRequest",
36+
"allDeclaredFields":true,
37+
"allDeclaredMethods":true
38+
},
39+
{
40+
"name":"com.google.api.services.storage.model.StorageObject",
41+
"allDeclaredFields":true,
42+
"allDeclaredMethods":true
43+
},
44+
{
45+
"name":"com.google.common.util.concurrent.AbstractFuture",
46+
"fields":[
47+
{"name":"listeners", "allowUnsafeAccess":true},
48+
{"name":"value", "allowUnsafeAccess":true},
49+
{"name":"waiters", "allowUnsafeAccess":true}
50+
]
51+
},
52+
{
53+
"name":"com.google.common.util.concurrent.AbstractFuture$Waiter",
54+
"fields":[
55+
{"name":"next", "allowUnsafeAccess":true},
56+
{"name":"thread", "allowUnsafeAccess":true}
57+
]
58+
},
59+
{
60+
"name":"com.google.protobuf.ExtensionRegistry",
61+
"methods":[{"name":"getEmptyRegistry","parameterTypes":[] }]
62+
},
63+
{
64+
"name":"java.lang.Object",
65+
"allDeclaredFields":true
66+
},
67+
{
68+
"name":"java.lang.Throwable",
69+
"methods":[{"name":"addSuppressed","parameterTypes":["java.lang.Throwable"] }]
70+
},
71+
{
72+
"name":"java.lang.invoke.VarHandle"
73+
},
74+
{
75+
"name":"java.lang.reflect.AccessibleObject",
76+
"fields":[{"name":"override"}]
77+
},
78+
{
79+
"name":"java.nio.Buffer",
80+
"fields":[{"name":"address", "allowUnsafeAccess":true}]
81+
},
82+
{
83+
"name":"java.util.AbstractMap",
84+
"allDeclaredFields":true
85+
},
86+
{
87+
"name":"org.tensorflow.example.BytesList",
88+
"methods":[
89+
{"name":"getValue","parameterTypes":["int"] },
90+
{"name":"getValueCount","parameterTypes":[] },
91+
{"name":"getValueList","parameterTypes":[] },
92+
{"name":"newBuilder","parameterTypes":[] }
93+
]
94+
},
95+
{
96+
"name":"org.tensorflow.example.BytesList$Builder",
97+
"methods":[
98+
{"name":"addValue","parameterTypes":["com.google.protobuf.ByteString"] },
99+
{"name":"clearValue","parameterTypes":[] },
100+
{"name":"getValue","parameterTypes":["int"] },
101+
{"name":"getValueCount","parameterTypes":[] },
102+
{"name":"getValueList","parameterTypes":[] },
103+
{"name":"setValue","parameterTypes":["int","com.google.protobuf.ByteString"] }
104+
]
105+
},
106+
{
107+
"name":"org.tensorflow.example.Example",
108+
"methods":[
109+
{"name":"getFeatures","parameterTypes":[] },
110+
{"name":"hasFeatures","parameterTypes":[] }
111+
]
112+
},
113+
{
114+
"name":"org.tensorflow.example.Example$Builder",
115+
"methods":[
116+
{"name":"clearFeatures","parameterTypes":[] },
117+
{"name":"getFeatures","parameterTypes":[] },
118+
{"name":"getFeaturesBuilder","parameterTypes":[] },
119+
{"name":"hasFeatures","parameterTypes":[] },
120+
{"name":"setFeatures","parameterTypes":["org.tensorflow.example.Features"] }
121+
]
122+
},
123+
{
124+
"name":"org.tensorflow.example.Feature",
125+
"methods":[
126+
{"name":"getBytesList","parameterTypes":[] },
127+
{"name":"getFloatList","parameterTypes":[] },
128+
{"name":"getInt64List","parameterTypes":[] },
129+
{"name":"getKindCase","parameterTypes":[] }
130+
]
131+
},
132+
{
133+
"name":"org.tensorflow.example.Feature$Builder",
134+
"methods":[
135+
{"name":"clearBytesList","parameterTypes":[] },
136+
{"name":"clearFloatList","parameterTypes":[] },
137+
{"name":"clearInt64List","parameterTypes":[] },
138+
{"name":"clearKind","parameterTypes":[] },
139+
{"name":"getBytesList","parameterTypes":[] },
140+
{"name":"getBytesListBuilder","parameterTypes":[] },
141+
{"name":"getFloatList","parameterTypes":[] },
142+
{"name":"getFloatListBuilder","parameterTypes":[] },
143+
{"name":"getInt64List","parameterTypes":[] },
144+
{"name":"getInt64ListBuilder","parameterTypes":[] },
145+
{"name":"getKindCase","parameterTypes":[] },
146+
{"name":"setBytesList","parameterTypes":["org.tensorflow.example.BytesList"] },
147+
{"name":"setFloatList","parameterTypes":["org.tensorflow.example.FloatList"] },
148+
{"name":"setInt64List","parameterTypes":["org.tensorflow.example.Int64List"] }
149+
]
150+
},
151+
{
152+
"name":"org.tensorflow.example.Features",
153+
"methods":[
154+
{"name":"getDefaultInstance","parameterTypes":[] },
155+
{"name":"newBuilder","parameterTypes":[] }
156+
]
157+
},
158+
{
159+
"name":"org.tensorflow.example.FloatList",
160+
"methods":[
161+
{"name":"getValue","parameterTypes":["int"] },
162+
{"name":"getValueCount","parameterTypes":[] },
163+
{"name":"getValueList","parameterTypes":[] },
164+
{"name":"newBuilder","parameterTypes":[] }
165+
]
166+
},
167+
{
168+
"name":"org.tensorflow.example.FloatList$Builder",
169+
"methods":[
170+
{"name":"addValue","parameterTypes":["float"] },
171+
{"name":"clearValue","parameterTypes":[] },
172+
{"name":"getValue","parameterTypes":["int"] },
173+
{"name":"getValueCount","parameterTypes":[] },
174+
{"name":"getValueList","parameterTypes":[] },
175+
{"name":"setValue","parameterTypes":["int","float"] }
176+
]
177+
},
178+
{
179+
"name":"org.tensorflow.example.Int64List",
180+
"methods":[
181+
{"name":"getValue","parameterTypes":["int"] },
182+
{"name":"getValueCount","parameterTypes":[] },
183+
{"name":"getValueList","parameterTypes":[] },
184+
{"name":"newBuilder","parameterTypes":[] }
185+
]
186+
},
187+
{
188+
"name":"org.tensorflow.example.Int64List$Builder",
189+
"methods":[
190+
{"name":"addValue","parameterTypes":["long"] },
191+
{"name":"clearValue","parameterTypes":[] },
192+
{"name":"getValue","parameterTypes":["int"] },
193+
{"name":"getValueCount","parameterTypes":[] },
194+
{"name":"getValueList","parameterTypes":[] },
195+
{"name":"setValue","parameterTypes":["int","long"] }
196+
]
197+
},
198+
{
199+
"name":"scala.Symbol"
200+
},
201+
{
202+
"name":"sun.misc.Unsafe",
203+
"allDeclaredFields":true,
204+
"methods":[
205+
{"name":"arrayBaseOffset","parameterTypes":["java.lang.Class"] },
206+
{"name":"arrayIndexScale","parameterTypes":["java.lang.Class"] },
207+
{"name":"copyMemory","parameterTypes":["long","long","long"] },
208+
{"name":"copyMemory","parameterTypes":["java.lang.Object","long","java.lang.Object","long","long"] },
209+
{"name":"getBoolean","parameterTypes":["java.lang.Object","long"] },
210+
{"name":"getByte","parameterTypes":["long"] },
211+
{"name":"getByte","parameterTypes":["java.lang.Object","long"] },
212+
{"name":"getDouble","parameterTypes":["java.lang.Object","long"] },
213+
{"name":"getFloat","parameterTypes":["java.lang.Object","long"] },
214+
{"name":"getInt","parameterTypes":["long"] },
215+
{"name":"getInt","parameterTypes":["java.lang.Object","long"] },
216+
{"name":"getLong","parameterTypes":["long"] },
217+
{"name":"getLong","parameterTypes":["java.lang.Object","long"] },
218+
{"name":"getObject","parameterTypes":["java.lang.Object","long"] },
219+
{"name":"objectFieldOffset","parameterTypes":["java.lang.reflect.Field"] },
220+
{"name":"putBoolean","parameterTypes":["java.lang.Object","long","boolean"] },
221+
{"name":"putByte","parameterTypes":["long","byte"] },
222+
{"name":"putByte","parameterTypes":["java.lang.Object","long","byte"] },
223+
{"name":"putDouble","parameterTypes":["java.lang.Object","long","double"] },
224+
{"name":"putFloat","parameterTypes":["java.lang.Object","long","float"] },
225+
{"name":"putInt","parameterTypes":["long","int"] },
226+
{"name":"putInt","parameterTypes":["java.lang.Object","long","int"] },
227+
{"name":"putLong","parameterTypes":["long","long"] },
228+
{"name":"putLong","parameterTypes":["java.lang.Object","long","long"] },
229+
{"name":"putObject","parameterTypes":["java.lang.Object","long","java.lang.Object"] }
230+
]
231+
}
232+
]

‎cli/src/main/scala/tfr/Cli.scala

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,8 @@
1616
*/
1717
package tfr
1818

19-
import java.io.FileInputStream
20-
2119
import caseapp._
22-
import cats.effect.{IO, Resource, ContextShift}
20+
import cats.effect.{IO, ContextShift}
2321
import fs2._
2422
import org.tensorflow.example.Example
2523
import tfr.instances._
@@ -41,12 +39,10 @@ object Cli extends CaseApp[Options] {
4139
override def run(options: Options, args: RemainingArgs): Unit = {
4240
val resources = args.remaining match {
4341
case Nil =>
44-
Stream.resource(Resource.fromAutoCloseable(IO.delay(System.in))) :: Nil
42+
Stream.resource(Resources.stdin[IO]) :: Nil
4543
case l =>
4644
l.iterator.map { path =>
47-
Stream.resource(Resource.fromAutoCloseable(IO.delay {
48-
new FileInputStream(new java.io.File(path))
49-
}))
45+
Stream.resource(Resources.file[IO](path))
5046
}.toList
5147
}
5248

‎core/src/main/scala/tfr/Resources.scala

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* Copyright 2020 Spotify AB.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing,
11+
* software distributed under the License is distributed on an
12+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
13+
* KIND, either express or implied. See the License for the
14+
* specific language governing permissions and limitations
15+
* under the License.
16+
*/
17+
package tfr
18+
19+
import cats.effect.Resource
20+
import cats.effect.Sync
21+
import com.google.cloud.storage.StorageOptions
22+
import java.net.URI
23+
import java.nio.channels.Channels
24+
import java.nio.file.Files
25+
import java.nio.file.Paths
26+
27+
object Resources {
28+
29+
final def stdin[F[_]: Sync] =
30+
Resource.fromAutoCloseable(Sync[F].delay(System.in))
31+
32+
final def file[F[_]: Sync](path: String) =
33+
Resource.fromAutoCloseable(Sync[F].delay {
34+
URI.create(path) match {
35+
case gcsUri if gcsUri.getScheme == "gs" =>
36+
val service = StorageOptions.getDefaultInstance().getService()
37+
val blobPath = gcsUri.getPath().tail match {
38+
case s if s.endsWith("/") => s.init
39+
case s => s
40+
}
41+
val readChannel = service.reader(gcsUri.getHost(), blobPath)
42+
Channels.newInputStream(readChannel)
43+
case uri =>
44+
Files.newInputStream(Paths.get(uri.getPath()))
45+
}
46+
})
47+
48+
}

‎project/Dependencies.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,5 @@ object Dependencies {
99
lazy val fs2Io = "co.fs2" %% "fs2-io" % "2.2.1"
1010
lazy val caseApp = "com.github.alexarchambault" %% "case-app" % "2.0.0-M10"
1111
lazy val kindProjector = "org.typelevel" %% "kind-projector" % "0.11.0" cross CrossVersion.full
12+
lazy val gcs = "com.google.cloud" % "google-cloud-storage" % "1.103.1"
1213
}

0 commit comments

Comments
 (0)