Skip to content

Commit a980860

Browse files
cmatthewcmatthew
committed
Multi-table support for Tableau
Run info and samples are extracted as separate tables. Each is inserted into Tableau batched. This makes the extracts much smaller and faster.
1 parent a5501f1 commit a980860

File tree

1 file changed

+164
-75
lines changed

1 file changed

+164
-75
lines changed

‎lnt/server/ui/static/lnt_tableau.js

Lines changed: 164 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -64,107 +64,196 @@
6464
var field_info = getValue(ts_url + "/fields/");
6565

6666
var fields = field_info.fields;
67-
var cols = [];
68-
cols.push({
67+
var sample_cols = [];
68+
var run_cols = [];
69+
70+
run_cols.push({
6971
id: "machine_name",
7072
alias: "Machine Name",
7173
dataType: tableau.dataTypeEnum.string
72-
});
73-
cols.push({
74-
id: "run_id",
75-
alias: "Run ID",
76-
dataType: tableau.dataTypeEnum.int
77-
});
78-
cols.push({
79-
id: "run_order",
80-
alias: "Run Order",
81-
dataType: tableau.dataTypeEnum.string
82-
});
83-
cols.push({
84-
id: "run_date",
85-
alias: "Run DateTime",
86-
dataType: tableau.dataTypeEnum.datetime
87-
});
88-
cols.push({
89-
id: "test_name",
90-
alias: "Test",
91-
dataType: tableau.dataTypeEnum.string
92-
});
74+
});
9375

94-
fields.forEach(function(field) {
95-
cols.push({
96-
id: field.column_name,
97-
alias: field.column_name,
98-
dataType: col_type_mapper[field.column_name]
99-
});
76+
run_cols.push({
77+
id: "run_id",
78+
alias: "Run ID",
79+
dataType: tableau.dataTypeEnum.int
80+
});
81+
sample_cols.push({
82+
id: "run_id",
83+
alias: "Run ID",
84+
dataType: tableau.dataTypeEnum.int,
85+
filterable: true,
86+
foreignKey: {
87+
"tableId": "measurement_data",
88+
"columnId": "run_id"
89+
}
90+
});
91+
run_cols.push({
92+
id: "run_order",
93+
alias: "Run Order",
94+
dataType: tableau.dataTypeEnum.string
95+
});
96+
run_cols.push({
97+
id: "run_date",
98+
alias: "Run DateTime",
99+
dataType: tableau.dataTypeEnum.datetime
100+
});
101+
sample_cols.push({
102+
id: "test_name",
103+
alias: "Test",
104+
dataType: tableau.dataTypeEnum.string
105+
});
106+
107+
fields.forEach(function(field) {
108+
sample_cols.push({
109+
id: field.column_name,
110+
alias: field.column_name,
111+
dataType: col_type_mapper[field.column_name]
100112
});
101-
var tableSchema = {
102-
id: "lnt_machine_feed",
103-
alias: "Performance Data from " + field_info.generated_by,
104-
columns: cols,
113+
});
114+
115+
var measurementsSchema = {
116+
id: "measurement_data",
117+
alias: "Measurement Data",
118+
columns: sample_cols,
119+
incrementColumnId: "run_id",
120+
joinOnly: true
121+
};
122+
123+
var runSchema = {
124+
id: "run_information",
125+
alias: "Run Information",
126+
columns: run_cols,
105127
incrementColumnId: "run_id"
128+
};
129+
130+
var standardConnection = {
131+
"alias": "Measurements with Run Info",
132+
"tables": [{
133+
"id": "run_information",
134+
"alias": "Run Information"
135+
}, {
136+
"id": "measurement_data",
137+
"alias": "Measurements"
138+
}],
139+
"joins": [{
140+
"left": {
141+
"tableAlias": "Run Information",
142+
"columnId": "run_id"
143+
},
144+
"right": {
145+
"tableAlias": "Measurements",
146+
"columnId": "run_id"
147+
},
148+
"joinType": "inner"
149+
}]
106150
};
107-
schemaCallback([tableSchema]);
151+
//
152+
schemaCallback([runSchema, measurementsSchema],[standardConnection] );
108153
}
109154

110155

111156
// Download the data.
112157
myConnector.getData = function (table, doneCallback) {
158+
// How often (in terms of fetches) to report progress.
159+
var progress_batch_size = 100;
113160
var last_run_id = parseInt(table.incrementValue || 0);
161+
if (table.tableInfo.id === "run_information") {
162+
// Collect the Run Infos table.
163+
// Get latest machines.
164+
var search_info = JSON.parse(tableau.connectionData);
165+
var machine_names = get_matching_machines(search_info.machine_regexp);
166+
if (machine_names.length === 0) {
167+
tableau.abortWithError("Did not match any machine names matching: " +
168+
search_info.machine_regexp);
169+
} else {
170+
tableau.reportProgress("Found " + machine_names.length +
171+
" machines to fetch.");
172+
}
114173

115-
// Get latest machines.
116-
var search_info = JSON.parse(tableau.connectionData);
117-
var machine_names = get_matching_machines(search_info.machine_regexp);
118-
if (machine_names.length === 0) {
119-
tableau.abortWithError("Did not match any machine names matching: " +
120-
search_info.machine_regexp);
121-
} else {
122-
tableau.reportProgress("Found " + machine_names.length +
123-
" machines to fetch.");
124-
}
174+
// Get runs for each of the filtered machines in batches.
175+
var tableData = [];
176+
// It is faster to submit in batches.
177+
var submission_batch_size = 10000;
178+
var total_fetched = 1;
179+
machine_names.forEach(function (machine) {
180+
var machines_run_data = getValue(ts_url + "/machines/" + machine.id)
181+
182+
machines_run_data.runs.forEach(function(run_info){
183+
// Incremental support.
184+
if (run_info.id <= last_run_id) {
185+
return;
186+
}
187+
188+
var date_str = run_info.end_time;
189+
var run_date = new Date(date_str);
190+
tableData.push({run_id: run_info.id,
191+
machine_name: machines_run_data.machine.name,
192+
run_order: run_info[run_info.order_by],
193+
run_date: run_date});
194+
195+
if (total_fetched % submission_batch_size == 0) {
196+
table.appendRows(tableData);
197+
tableData.length = 0; // Drop the submitted rows.
198+
}
199+
total_fetched = total_fetched + 1;
200+
});
201+
});
202+
table.appendRows(tableData);
203+
} else if (table.tableInfo.id === "measurement_data") {
204+
// Collect Sample data.
205+
var filterValues = table.filterValues;
206+
207+
if (!table.isJoinFiltered) {
208+
tableau.abortWithError("The table must be filtered first.");
209+
return;
210+
}
211+
212+
if (filterValues.length === 0) {
213+
doneCallback();
214+
return;
215+
}
125216

126-
machine_names.forEach(function (machine) {
127-
var url = ts_url + "/machines/" + machine.id;
128-
var machine_info = getValue(url);
129-
var machine_name = machine_info.machine.name;
130217
var tableData = [];
218+
var total_fetched = 1;
219+
// Processing by batch is much faster; however, there is a
220+
// 128mb limit to the JS interpreter in Tableau.
221+
// This is a guess with some envelope math.
222+
var submission_batch_size = 10000;
131223

132-
machine_info.runs.forEach(function(run, index) {
133-
var run_data;
134-
var runs_total = machine_info.runs.length;
135-
// Run based incremental refresh. If we have already seen data, skip it.
136-
if (run.id <= last_run_id) {
137-
return;
224+
for (var i in filterValues) {
225+
226+
var run_id = filterValues[i];
227+
if (run_id <= last_run_id) {
228+
return;
229+
}
230+
231+
if (total_fetched % progress_batch_size == 1) {
232+
var next_run_max = total_fetched + progress_batch_size;
233+
var status_msg = "Getting Runs: " + run_id +
234+
" (" + total_fetched + "-" + next_run_max + " of " + filterValues.length + ").";
235+
tableau.reportProgress(status_msg);
138236
}
139237

140-
var status_msg = "Getting Machine: " + machine_name
141-
+ " Run: " + run.id
142-
+ " (" + (index + 1) + "/" + runs_total + ")";
143-
144-
tableau.reportProgress(status_msg);
145-
run_data = getValue(ts_url + "/runs/" + run.id);
146-
147-
var date_str = run_data.run.end_time;
148-
var run_date = new Date(date_str);
149-
var derived_run_data = {
150-
"machine_name": machine_name,
151-
"run_id": run.id,
152-
"run_order": run[run.order_by],
153-
"run_date": run_date
154-
};
238+
run_data = getValue(ts_url + "/runs/" + run_id);
239+
155240
run_data.tests.forEach(function (element) {
156241
element.test_name = element.name;
157242
delete element.name;
158-
var data = Object.assign({}, derived_run_data, element);
159-
tableData.push(data);
243+
tableData.push(element);
160244

161245
});
246+
if (tableData.length > submission_batch_size) {
247+
table.appendRows(tableData);
248+
tableData.length = 0; // Drop the submitted rows.
249+
}
250+
total_fetched = total_fetched + 1;
162251
run_data = null;
163-
});
164-
252+
}
165253
table.appendRows(tableData);
166-
167-
});
254+
} else {
255+
throw new Error("Unexpected table id " + table.tableInfo.id)
256+
}
168257
doneCallback();
169258
};
170259

0 commit comments

Comments
 (0)