Skip to content

Commit a98b940

Browse files
zhangpengshanzhangpengshan
committed
quick fix some ut issue
1 parent 14b92bc commit a98b940

18 files changed

+29
-1097
lines changed

‎src/main/java/ml/shifu/shifu/pig/ShifuPigStorage.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ public void setLocation(String location, Job job) throws IOException {
6666
shifuStorage.setLocation(location, job);
6767
}
6868

69+
@SuppressWarnings("rawtypes")
6970
@Override
7071
public InputFormat getInputFormat() {
7172
return shifuStorage.getInputFormat();
@@ -96,6 +97,7 @@ public List<OperatorSet> getFeatures() {
9697
return shifuStorage.getFeatures();
9798
}
9899

100+
@SuppressWarnings("rawtypes")
99101
@Override
100102
public void prepareToWrite(RecordWriter writer) {
101103
shifuStorage.prepareToWrite(writer);

‎src/test/java/ml/shifu/shifu/ShifuCLITest.java

Lines changed: 6 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,6 @@
2121
import java.io.IOException;
2222
import java.io.InputStreamReader;
2323
import java.util.Arrays;
24-
import java.util.HashMap;
25-
import java.util.Map;
26-
27-
import ml.shifu.shifu.container.obj.RawSourceData.SourceType;
28-
import ml.shifu.shifu.fs.ShifuFileUtils;
29-
import ml.shifu.shifu.util.CommonUtils;
30-
import ml.shifu.shifu.util.Constants;
31-
import ml.shifu.shifu.util.Environment;
3224

3325
import org.apache.commons.io.FileUtils;
3426
import org.easymock.EasyMock;
@@ -41,6 +33,12 @@
4133
import org.testng.annotations.ObjectFactory;
4234
import org.testng.annotations.Test;
4335

36+
import ml.shifu.shifu.container.obj.RawSourceData.SourceType;
37+
import ml.shifu.shifu.fs.ShifuFileUtils;
38+
import ml.shifu.shifu.util.CommonUtils;
39+
import ml.shifu.shifu.util.Constants;
40+
import ml.shifu.shifu.util.Environment;
41+
4442
/**
4543
* ManagerTest class
4644
*/
@@ -212,7 +210,6 @@ public void testTrainModel() throws Exception {
212210
FileUtils.deleteQuietly(tmpColumn);
213211
FileUtils.deleteDirectory(new File("tmp"));
214212
FileUtils.deleteDirectory(new File("models"));
215-
216213
}
217214

218215
// @Test
@@ -332,33 +329,6 @@ public void testSplit() {
332329
System.out.println(Arrays.asList("01Jun2015^0.3456".split("\\^")));
333330
}
334331

335-
@Test
336-
public void testRebin() throws Exception {
337-
File originModel = new File("src/test/resources/example/binning-data/rebin/ModelConfig.json");
338-
File tmpModel = new File("ModelConfig.json");
339-
FileUtils.copyFile(originModel, tmpModel);
340-
341-
File originColumn = new File("src/test/resources/example/binning-data/rebin/ColumnConfig.json");
342-
File tmpColumn = new File("ColumnConfig.json");
343-
FileUtils.copyFile(originColumn, tmpColumn);
344-
345-
File originConfDir = new File("src/test/resources/example/binning-data/rebin/columns");
346-
File tmpConfDir = new File("columns");
347-
FileUtils.copyDirectory(originConfDir, tmpConfDir);
348-
349-
Map<String, Object> params = new HashMap<String, Object>();
350-
params.put(Constants.IS_REBIN, true);
351-
params.put(Constants.IV_KEEP_RATIO, "0.975");
352-
params.put(Constants.MINIMUM_BIN_INST_CNT, "2000");
353-
ShifuCLI.calModelStats(params);
354-
Assert.assertTrue(new File("tmp/ColumnConfig.json").exists());
355-
356-
FileUtils.deleteQuietly(tmpModel);
357-
FileUtils.deleteQuietly(tmpColumn);
358-
FileUtils.deleteDirectory(tmpConfDir);
359-
FileUtils.deleteDirectory(new File("tmp"));
360-
}
361-
362332
@Test
363333
public void testLastIndexOf() {
364334
String text = "aa^bb^cc";

‎src/test/java/ml/shifu/shifu/core/binning/CategoricalBinningTest.java

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,6 @@
1717
*/
1818
package ml.shifu.shifu.core.binning;
1919

20-
import junit.framework.Assert;
21-
22-
import org.testng.annotations.Test;
23-
2420
/**
2521
* CategoricalBinningTest class
2622
*
@@ -29,12 +25,4 @@
2925
*/
3026
public class CategoricalBinningTest {
3127

32-
@Test
33-
public void testSerialObject() {
34-
CategoricalBinning binning = new CategoricalBinning(10, 5000);
35-
String binStr = binning.objToString();
36-
String[] fieldArr = binStr.split(Character.toString(AbstractBinning.FIELD_SEPARATOR), -1);
37-
Assert.assertTrue(fieldArr.length == 6);
38-
}
39-
4028
}

‎src/test/java/ml/shifu/shifu/core/binning/ColumnConfigDynamicBinningTest.java

Lines changed: 0 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,14 @@
11
package ml.shifu.shifu.core.binning;
22

33

4-
import ml.shifu.shifu.container.obj.ColumnConfig;
5-
import ml.shifu.shifu.container.obj.RawSourceData;
6-
import ml.shifu.shifu.core.binning.obj.AbstractBinInfo;
7-
import ml.shifu.shifu.util.CommonUtils;
84
import org.testng.Assert;
95
import org.testng.annotations.Test;
106

11-
import java.io.IOException;
12-
import java.util.List;
13-
147
/**
158
* Created by zhanhu on 5/8/17.
169
*/
1710
public class ColumnConfigDynamicBinningTest {
1811

19-
@Test
20-
public void testCategoricalConfigDynamicBinning() throws IOException {
21-
List<ColumnConfig> columnConfigList = CommonUtils.loadColumnConfigList(
22-
"src/test/resources/example/binning-data/ColumnConfig.json", RawSourceData.SourceType.LOCAL);
23-
ColumnConfigDynamicBinning ccdb = new ColumnConfigDynamicBinning(
24-
columnConfigList.get(0), 0, 0.99, 0);
25-
List<AbstractBinInfo> binInfos= ccdb.run();
26-
Assert.assertEquals(binInfos.size(), 11);
27-
}
28-
29-
@Test
30-
public void testNumericalConfigDynamicBinning() throws IOException {
31-
List<ColumnConfig> columnConfigList = CommonUtils.loadColumnConfigList(
32-
"src/test/resources/example/binning-data/ColumnConfig.json", RawSourceData.SourceType.LOCAL);
33-
ColumnConfigDynamicBinning ccdb = new ColumnConfigDynamicBinning(
34-
columnConfigList.get(2), 0, 0.99, 2000);
35-
List<AbstractBinInfo> binInfos= ccdb.run();
36-
Assert.assertEquals(binInfos.size(), 41);
37-
}
38-
39-
@Test
40-
public void testNumericalConfigDynamicBinning2() throws IOException {
41-
List<ColumnConfig> columnConfigList = CommonUtils.loadColumnConfigList(
42-
"src/test/resources/example/binning-data/ColumnConfig.json", RawSourceData.SourceType.LOCAL);
43-
ColumnConfigDynamicBinning ccdb = new ColumnConfigDynamicBinning(
44-
columnConfigList.get(3), 0, 0.99, 2000);
45-
List<AbstractBinInfo> binInfos= ccdb.run();
46-
Assert.assertEquals(binInfos.size(), 17);
47-
}
48-
4912
@Test
5013
public void testIvReduce() {
5114
double iv1 = calcualte(0, 28);

‎src/test/java/ml/shifu/shifu/core/binning/DynamicCategoricalBinTest.java

Lines changed: 0 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1,89 +1,8 @@
11
package ml.shifu.shifu.core.binning;
22

3-
import ml.shifu.shifu.core.binning.obj.AbstractBinInfo;
4-
import ml.shifu.shifu.core.binning.obj.CategoricalBinInfo;
5-
import org.apache.commons.io.IOUtils;
6-
import org.apache.commons.lang.StringUtils;
7-
import org.junit.Test;
8-
import org.testng.Assert;
9-
10-
import java.io.IOException;
11-
import java.util.ArrayList;
12-
import java.util.Collections;
13-
import java.util.List;
14-
153
/**
164
* Created by zhanhu on 4/18/17.
175
*/
186
public class DynamicCategoricalBinTest {
197

20-
@Test
21-
public void testDynamicCategoricalBin() throws IOException {
22-
List<AbstractBinInfo> categoricalBinInfoList = loadTestData();
23-
Collections.sort(categoricalBinInfoList);
24-
25-
AutoDynamicBinning binning = new AutoDynamicBinning(2);
26-
List<AbstractBinInfo> finalBins = binning.merge(categoricalBinInfoList);
27-
28-
Assert.assertEquals(finalBins.size(), 2);
29-
30-
List<String> categoricalVals = new ArrayList<String>();
31-
List<Long> negativeCnts = new ArrayList<Long>();
32-
List<Long> positiveCnts = new ArrayList<Long>();
33-
List<Double> positiveRates = new ArrayList<Double>();
34-
35-
for ( int i = 0; i < finalBins.size(); i ++ ) {
36-
CategoricalBinInfo binInfo = (CategoricalBinInfo)finalBins.get(i);
37-
categoricalVals.add("\"" + StringUtils.join(binInfo.getValues(), '^') + "\"");
38-
negativeCnts.add(binInfo.getNegativeCnt());
39-
positiveCnts.add(binInfo.getPositiveCnt());
40-
positiveRates.add(binInfo.getPositiveRate());
41-
}
42-
43-
System.out.println(StringUtils.join(categoricalVals, ','));
44-
System.out.println(StringUtils.join(negativeCnts, ','));
45-
System.out.println(StringUtils.join(positiveCnts, ','));
46-
System.out.println(StringUtils.join(positiveRates, ','));
47-
}
48-
49-
private List<AbstractBinInfo> loadTestData() throws IOException {
50-
List<String> lines = IOUtils.readLines(DynamicCategoricalBinTest.class
51-
.getResourceAsStream("/example/binning-data/categorical-binning"));
52-
53-
String[] categories = lines.get(0)
54-
.replaceAll("^.* \\[", "")
55-
.replaceAll("].*$", "")
56-
.replaceAll("\"", "").trim().split(",");
57-
58-
String[] binPosCounts = lines.get(2)
59-
.replaceAll("^.* \\[", "")
60-
.replaceAll("].*$", "")
61-
.replaceAll("\"", "").trim().split(",");
62-
63-
String[] binNegCounts = lines.get(1)
64-
.replaceAll("^.* \\[", "")
65-
.replaceAll("].*$", "")
66-
.replaceAll("\"", "").trim().split(",");
67-
68-
@SuppressWarnings("unused")
69-
String[] positiveRates = lines.get(3)
70-
.replaceAll("^.* \\[", "")
71-
.replaceAll("].*$", "")
72-
.replaceAll("\"", "").trim().split(",");
73-
74-
List<AbstractBinInfo> categoricalBinInfos = new ArrayList<AbstractBinInfo>();
75-
for ( int i = 0; i < categories.length; i ++ ) {
76-
CategoricalBinInfo binInfo = new CategoricalBinInfo();
77-
List<String> values = new ArrayList<String>();
78-
values.add(categories[i].trim());
79-
binInfo.setValues(values);
80-
81-
binInfo.setPositiveCnt(Long.parseLong(binPosCounts[i].trim()));
82-
binInfo.setNegativeCnt(Long.parseLong(binNegCounts[i].trim()));
83-
84-
categoricalBinInfos.add(binInfo);
85-
}
86-
87-
return categoricalBinInfos;
88-
}
898
}

‎src/test/java/ml/shifu/shifu/core/binning/EqualIntervalBinningTest.java

Lines changed: 0 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -17,79 +17,11 @@
1717
*/
1818
package ml.shifu.shifu.core.binning;
1919

20-
import java.io.IOException;
21-
import java.io.InputStream;
22-
import java.util.List;
23-
import java.util.Random;
24-
import java.util.zip.GZIPInputStream;
25-
26-
import junit.framework.Assert;
27-
28-
import ml.shifu.shifu.container.obj.ModelConfig;
29-
import ml.shifu.shifu.container.obj.RawSourceData;
30-
import ml.shifu.shifu.util.CommonUtils;
31-
import org.apache.commons.io.IOUtils;
32-
import org.testng.annotations.Test;
33-
3420
/**
3521
* EqualIntervalBinningTest class
3622
*
3723
* @Oct 27, 2014
3824
*/
3925
public class EqualIntervalBinningTest {
4026

41-
@Test
42-
public void testBinning() {
43-
Random rd = new Random(System.currentTimeMillis());
44-
45-
EqualIntervalBinning binning = new EqualIntervalBinning(10);
46-
for (int i = 0; i < 10000; i++) {
47-
binning.addData(Integer.toString(rd.nextInt() % 1000));
48-
}
49-
50-
System.out.println(binning.getDataBin());
51-
}
52-
53-
54-
@Test
55-
public void tesGussiantBinning() {
56-
Random rd = new Random(System.currentTimeMillis());
57-
58-
EqualIntervalBinning binning = new EqualIntervalBinning(10);
59-
for (int i = 0; i < 10000; i++) {
60-
binning.addData(Double.toString(rd.nextGaussian() % 1000));
61-
}
62-
63-
System.out.println(binning.getDataBin());
64-
}
65-
66-
@Test
67-
public void testSerialObject() {
68-
EqualIntervalBinning binning = new EqualIntervalBinning();
69-
String binStr = binning.objToString();
70-
String[] fieldArr = binStr.split(Character.toString(AbstractBinning.FIELD_SEPARATOR));
71-
Assert.assertTrue(fieldArr.length == 6);
72-
}
73-
74-
@Test
75-
public void testSmallBinning() throws IOException {
76-
ModelConfig modelConfig = CommonUtils.loadModelConfig("src/test/resources/example/inner_seg1_v15/ModelConfig.json", RawSourceData.SourceType.LOCAL);
77-
78-
EqualIntervalBinning inst = new EqualIntervalBinning(1000, modelConfig.getMissingOrInvalidValues());
79-
InputStream is = EqualIntervalBinningTest.class.getResourceAsStream("/example/inner_seg1_v15/dib_sample_fields.gz");
80-
GZIPInputStream gzis = new GZIPInputStream(is);
81-
82-
List<String> lines = IOUtils.readLines(gzis);
83-
for (String record : lines) {
84-
String[] fields = record.split("\\|");
85-
inst.addData(fields[0]);
86-
}
87-
88-
List<Double> boundaries = inst.getDataBin();
89-
System.out.println(boundaries);
90-
Assert.assertEquals(1001, boundaries.size());
91-
92-
IOUtils.closeQuietly(gzis);
93-
IOUtils.closeQuietly(is);
94-
}
9527
}

‎src/test/java/ml/shifu/shifu/core/binning/EqualPopulationBinningTest.java

Lines changed: 3 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -17,21 +17,18 @@
1717
*/
1818
package ml.shifu.shifu.core.binning;
1919

20-
import java.io.FileInputStream;
21-
import java.io.IOException;
2220
import java.util.ArrayList;
2321
import java.util.List;
2422
import java.util.Random;
2523

24+
import org.testng.Assert;
25+
import org.testng.annotations.Test;
26+
2627
import ml.shifu.shifu.container.obj.ColumnConfig;
2728
import ml.shifu.shifu.container.obj.ColumnType;
2829
import ml.shifu.shifu.container.obj.ModelConfig;
2930
import ml.shifu.shifu.container.obj.ModelStatsConf.BinningMethod;
3031

31-
import org.apache.commons.io.IOUtils;
32-
import org.testng.Assert;
33-
import org.testng.annotations.Test;
34-
3532
/**
3633
* EqualPopulationBinningTest class
3734
*
@@ -179,29 +176,4 @@ public void testSerialObject() {
179176
Assert.assertTrue(fieldArr.length == 5);
180177
}
181178

182-
@Test
183-
public void testUsageAge() throws IOException {
184-
EqualPopulationBinning binning = new EqualPopulationBinning(10);
185-
List<String> usageList = IOUtils.readLines(new FileInputStream("src/test/resources/example/binning-data/usage_age.txt"));
186-
187-
for ( String data : usageList ) {
188-
binning.addData(data);
189-
}
190-
191-
List<Double> binBoundary = binning.getDataBin();
192-
Assert.assertTrue(binBoundary.size() > 1);
193-
}
194-
195-
@Test
196-
public void testReturn180d() throws IOException {
197-
EqualPopulationBinning binning = new EqualPopulationBinning(10);
198-
List<String> usageList = IOUtils.readLines(new FileInputStream("src/test/resources/example/binning-data/return_lt_180d_amt.txt"));
199-
200-
for ( String data : usageList ) {
201-
binning.addData(data);
202-
}
203-
204-
List<Double> binBoundary = binning.getDataBin();
205-
Assert.assertTrue(binBoundary.size() > 1);
206-
}
207179
}

0 commit comments

Comments
 (0)