Skip to content

Commit 156c111

Browse files
igorbernstein2igorbernstein2
authored
feat: add bigtable-hbase-2x-mapreduce artifact (#4343)
This is a copy of bigtable-hbase-1-mapreduce, but with dependencies adjusted to hbase-mapreduce and the shaded assemblies removed. The assemblies have been removed because in a future iteration, customers will be expected to provide their own version hbase deps Change-Id: Ib3705bfa535a1dde9a9dce9446c311fcca4a6336 Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://togithub.com/googleapis/java-bigtable-hbase/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Fixes #<issue_number_goes_here> ☕️ If you write sample code, please follow the [samples format]( https://togithub.com/GoogleCloudPlatform/java-docs-samples/blob/main/SAMPLE_FORMAT.md).
1 parent 7be62ea commit 156c111

File tree

21 files changed

+3619
-4
lines changed

21 files changed

+3619
-4
lines changed

‎bigtable-hbase-2.x-parent/bigtable-hbase-2.x-mapreduce/pom.xml

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Copyright 2015 Google LLC
4+
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
-->
17+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
18+
<modelVersion>4.0.0</modelVersion>
19+
20+
<parent>
21+
<groupId>com.google.cloud.bigtable</groupId>
22+
<artifactId>bigtable-hbase-2.x-parent</artifactId>
23+
<version>2.14.0-SNAPSHOT</version> <!-- {x-version-update:bigtable-client-parent:current} -->
24+
</parent>
25+
26+
<artifactId>bigtable-hbase-2.x-mapreduce</artifactId>
27+
<packaging>jar</packaging>
28+
<name>${project.groupId}:${project.artifactId}</name>
29+
<description>
30+
This project contains tweaks to the hbase 2.* map reduce jobs that work for
31+
bigtable.
32+
Specifically, HBase's Import M/R job has ZooKeeper referrence which needed
33+
to be removed
34+
in order to work with Bigtable.
35+
</description>
36+
37+
<!-- NOTE: this artifact is designed to be used alongside hbase-server via
38+
the bin/hbase script. Its primary intention is to produce a single jar that can
39+
be added to hbase's classpath to kick off mapreduce jobs targeted at bigtable.
40+
A secondary goal is for some java orchestration program to kick off the jobs.
41+
42+
Thus the dependencies here must be exactly:
43+
- bigtable-hbase-1.x-hadoop
44+
- any hbase provided jars
45+
46+
In the primary usecase we will shade bigtable-hbase-1.x-hadoop allowing the
47+
enduser to drop a single jar in the hbase classpath. In the secondary case,
48+
the enduser can explcitly add their own version hbase-server.
49+
-->
50+
51+
<dependencies>
52+
<!-- Primary Group -->
53+
<dependency>
54+
<groupId>org.apache.hbase</groupId>
55+
<artifactId>hbase-mapreduce</artifactId>
56+
<version>${hbase2.version}</version>
57+
<scope>provided</scope>
58+
</dependency>
59+
60+
<dependency>
61+
<groupId>${project.groupId}</groupId>
62+
<artifactId>bigtable-hbase-2.x-hadoop</artifactId>
63+
<version>2.14.0-SNAPSHOT</version> <!-- {x-version-update:bigtable-client-parent:current} -->
64+
<exclusions>
65+
<!-- we need hbase-server instead of hbase-client -->
66+
<exclusion>
67+
<groupId>org.apache.hbase</groupId>
68+
<artifactId>hbase-client</artifactId>
69+
</exclusion>
70+
71+
<!-- Workaround MNG-5899 & MSHADE-206. Maven >= 3.3.0 doesn't use the dependency reduced
72+
pom.xml files when invoking the build from a parent project. So we have to manually exclude
73+
the dependencies. Note that this works in conjunction with the manually promoted dependencies
74+
in bigtable-hbase-1.x-shaded/pom.xml -->
75+
<exclusion>
76+
<groupId>${project.groupId}</groupId>
77+
<artifactId>bigtable-hbase-2.x-shaded</artifactId>
78+
</exclusion>
79+
</exclusions>
80+
</dependency>
81+
82+
<!-- Test Group -->
83+
<dependency>
84+
<groupId>junit</groupId>
85+
<artifactId>junit</artifactId>
86+
<version>${junit.version}</version>
87+
<scope>test</scope>
88+
</dependency>
89+
<dependency>
90+
<groupId>com.google.truth</groupId>
91+
<artifactId>truth</artifactId>
92+
<version>${truth.version}</version>
93+
<scope>test</scope>
94+
<exclusions>
95+
<!-- exclusion to avoid conflict with hbase-mapreduce -->
96+
<exclusion>
97+
<groupId>com.google.guava</groupId>
98+
<artifactId>guava</artifactId>
99+
</exclusion>
100+
<exclusion>
101+
<groupId>com.google.errorprone</groupId>
102+
<artifactId>error_prone_annotations</artifactId>
103+
</exclusion>
104+
<exclusion>
105+
<groupId>org.checkerframework</groupId>
106+
<artifactId>checker-qual</artifactId>
107+
</exclusion>
108+
</exclusions>
109+
</dependency>
110+
<dependency>
111+
<groupId>org.mockito</groupId>
112+
<artifactId>mockito-core</artifactId>
113+
<version>${mockito.version}</version>
114+
<scope>test</scope>
115+
</dependency>
116+
<dependency>
117+
<groupId>com.google.cloud</groupId>
118+
<artifactId>google-cloud-bigtable-emulator-core</artifactId>
119+
<version>${google-cloud-bigtable-emulator.version}</version>
120+
<scope>test</scope>
121+
</dependency>
122+
<dependency>
123+
<groupId>org.apache.hbase</groupId>
124+
<artifactId>hbase-testing-util</artifactId>
125+
<version>${hbase2.version}</version>
126+
<scope>test</scope>
127+
<exclusions>
128+
<exclusion>
129+
<!-- Exclude conflict with hbase-server -->
130+
<groupId>org.slf4j</groupId>
131+
<artifactId>slf4j-log4j12</artifactId>
132+
</exclusion>
133+
</exclusions>
134+
</dependency>
135+
<dependency>
136+
<groupId>com.google.cloud.bigtable</groupId>
137+
<artifactId>bigtable-internal-test-helper</artifactId>
138+
<version>2.14.0-SNAPSHOT</version> <!-- {x-version-update:bigtable-client-parent:current} -->
139+
<scope>test</scope>
140+
</dependency>
141+
</dependencies>
142+
143+
<build>
144+
<plugins>
145+
<plugin>
146+
<groupId>org.codehaus.mojo</groupId>
147+
<artifactId>build-helper-maven-plugin</artifactId>
148+
<executions>
149+
<execution>
150+
<id>add-source</id>
151+
<phase>generate-sources</phase>
152+
<goals>
153+
<goal>add-source</goal>
154+
</goals>
155+
<configuration>
156+
<sources>
157+
<source>../../third_party/third_party_hbase_server/src/main/mapreduce/</source>
158+
</sources>
159+
</configuration>
160+
</execution>
161+
</executions>
162+
</plugin>
163+
<plugin>
164+
<groupId>org.apache.maven.plugins</groupId>
165+
<artifactId>maven-jar-plugin</artifactId>
166+
<configuration>
167+
<archive>
168+
<manifest>
169+
<mainClass>com.google.cloud.bigtable.mapreduce.Driver</mainClass>
170+
</manifest>
171+
</archive>
172+
</configuration>
173+
</plugin>
174+
175+
176+
<plugin>
177+
<groupId>com.google.cloud.bigtable.test</groupId>
178+
<artifactId>bigtable-build-helper</artifactId>
179+
<version>2.14.0-SNAPSHOT</version> <!-- {x-version-update:bigtable-client-parent:current} -->
180+
<executions>
181+
<execution>
182+
<id>verify-mirror-deps-hbase</id>
183+
<phase>verify</phase>
184+
<goals>
185+
<goal>verify-mirror-deps</goal>
186+
</goals>
187+
<configuration>
188+
<targetDependencies>
189+
<targetDependency>org.apache.hbase:hbase-mapreduce</targetDependency>
190+
</targetDependencies>
191+
</configuration>
192+
</execution>
193+
</executions>
194+
</plugin>
195+
</plugins>
196+
</build>
197+
</project>

‎bigtable-hbase-2.x-parent/bigtable-hbase-2.x-mapreduce/src/main/java/com/google/cloud/bigtable/mapreduce/Driver.java

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/*
2+
* Copyright 2015 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigtable.mapreduce;
17+
18+
import com.google.cloud.bigtable.mapreduce.hbasesnapshots.ImportHBaseSnapshotJob;
19+
import com.google.cloud.bigtable.mapreduce.validation.BigtableSyncTableJob;
20+
import org.apache.hadoop.classification.InterfaceStability.Evolving;
21+
import org.apache.hadoop.hbase.mapreduce.HashTable;
22+
import org.apache.hadoop.util.ProgramDriver;
23+
24+
/** Driver for bigtable mapreduce jobs. Select which to run by passing name of job to this main. */
25+
@Evolving
26+
public class Driver {
27+
28+
/**
29+
* main.
30+
*
31+
* @param args an array of {@link java.lang.String} objects.
32+
*/
33+
public static void main(String[] args) {
34+
ProgramDriver programDriver = new ProgramDriver();
35+
int exitCode = -1;
36+
try {
37+
programDriver.addClass(
38+
"export-table",
39+
Export.class,
40+
"A map/reduce program that exports a table to sequencefiles.");
41+
programDriver.addClass(
42+
"import-table",
43+
Import.class,
44+
"A map/reduce program that imports sequencefiles to a table.");
45+
programDriver.addClass(
46+
"import-snapshot",
47+
ImportHBaseSnapshotJob.class,
48+
"A map/reduce program that imports an hbase snapshot to a table.");
49+
programDriver.addClass(
50+
"hash-table",
51+
HashTable.class,
52+
"A map/reduce program that computes hashes on source and outputs to filesystem (or cloud storage).");
53+
programDriver.addClass(
54+
"sync-table",
55+
BigtableSyncTableJob.class,
56+
"A map/reduce program that computes hashes on target and compares with hashes from source.");
57+
programDriver.driver(args);
58+
exitCode = programDriver.run(args);
59+
} catch (Throwable e) {
60+
e.printStackTrace();
61+
}
62+
System.exit(exitCode);
63+
}
64+
}

‎bigtable-hbase-2.x-parent/bigtable-hbase-2.x-mapreduce/src/main/java/com/google/cloud/bigtable/mapreduce/Export.java

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigtable.mapreduce;
17+
18+
import java.io.IOException;
19+
import org.apache.hadoop.conf.Configuration;
20+
import org.apache.hadoop.mapreduce.Job;
21+
22+
public class Export {
23+
24+
public static Job createSubmittableJob(Configuration conf, String[] args) throws IOException {
25+
return com.google.cloud.bigtable.thirdparty.org.apache.hadoop.hbase.mapreduce.Export
26+
.createSubmittableJob(conf, args);
27+
}
28+
29+
public static void main(String[] args) throws Exception {
30+
com.google.cloud.bigtable.thirdparty.org.apache.hadoop.hbase.mapreduce.Export.main(args);
31+
}
32+
}

0 commit comments

Comments
 (0)