Skip to content

Commit 9bd6805

Browse files
smlindauersmlindauer
committed
Update custom kpi and model parameters example to include check on the performance job states.
1 parent 34fbfb9 commit 9bd6805

File tree

1 file changed

+31
-210
lines changed

1 file changed

+31
-210
lines changed

‎examples/pzmm_custom_kpi_model_parameters.ipynb

Lines changed: 31 additions & 210 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
},
2222
{
2323
"cell_type": "code",
24-
"execution_count": 1,
24+
"execution_count": null,
2525
"metadata": {},
2626
"outputs": [],
2727
"source": [
@@ -48,12 +48,12 @@
4848
"metadata": {},
4949
"source": [
5050
"### Setup\n",
51-
"This example is built to be run after the [PZMM Binary Classification Model Import](/pzmm_binary_classification_model_import.ipynb) example notebook is run."
51+
"This example is built to be run after the [PZMM Binary Classification Model Import](./pzmm_binary_classification_model_import.ipynb) example notebook is run."
5252
]
5353
},
5454
{
5555
"cell_type": "code",
56-
"execution_count": 2,
56+
"execution_count": null,
5757
"metadata": {},
5858
"outputs": [],
5959
"source": [
@@ -73,7 +73,7 @@
7373
},
7474
{
7575
"cell_type": "code",
76-
"execution_count": 3,
76+
"execution_count": null,
7777
"metadata": {},
7878
"outputs": [],
7979
"source": [
@@ -95,7 +95,7 @@
9595
},
9696
{
9797
"cell_type": "code",
98-
"execution_count": 4,
98+
"execution_count": null,
9999
"metadata": {},
100100
"outputs": [],
101101
"source": [
@@ -124,7 +124,7 @@
124124
},
125125
{
126126
"cell_type": "code",
127-
"execution_count": 5,
127+
"execution_count": null,
128128
"metadata": {},
129129
"outputs": [],
130130
"source": [
@@ -146,7 +146,7 @@
146146
},
147147
{
148148
"cell_type": "code",
149-
"execution_count": 6,
149+
"execution_count": null,
150150
"metadata": {},
151151
"outputs": [],
152152
"source": [
@@ -165,20 +165,9 @@
165165
},
166166
{
167167
"cell_type": "code",
168-
"execution_count": 7,
168+
"execution_count": null,
169169
"metadata": {},
170-
"outputs": [
171-
{
172-
"name": "stdout",
173-
"output_type": "stream",
174-
"text": [
175-
"1\n",
176-
"2\n",
177-
"3\n",
178-
"4\n"
179-
]
180-
}
181-
],
170+
"outputs": [],
182171
"source": [
183172
"from sasctl._services.cas_management import CASManagement as cas\n",
184173
"\n",
@@ -198,7 +187,7 @@
198187
},
199188
{
200189
"cell_type": "code",
201-
"execution_count": 8,
190+
"execution_count": null,
202191
"metadata": {},
203192
"outputs": [],
204193
"source": [
@@ -209,7 +198,7 @@
209198
},
210199
{
211200
"cell_type": "code",
212-
"execution_count": 9,
201+
"execution_count": null,
213202
"metadata": {},
214203
"outputs": [],
215204
"source": [
@@ -222,31 +211,27 @@
222211
"cell_type": "markdown",
223212
"metadata": {},
224213
"source": [
225-
"Once the performance defintion is run, it is possible to update the hyperparameter json file to include the KPIs that have been generated. This is not a necessary step, but could be helpful when analyzing which hyperparameters lead to better KPIs."
214+
"Once the performance definition is run, it is possible to update the hyperparameter json file to include the KPIs that have been generated. This is not a necessary step, but could be helpful when analyzing which hyperparameters lead to better KPIs. Note that you need the performance jobs to have created the MM_STD performance table before running the `update_kpis()` method below."
226215
]
227216
},
228217
{
229218
"cell_type": "code",
230-
"execution_count": 10,
231-
"metadata": {},
232-
"outputs": [
233-
{
234-
"name": "stderr",
235-
"output_type": "stream",
236-
"text": [
237-
"~\\sasctl\\pzmm\\model_parameters.py:251: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.\n",
238-
" if pd.__version__ >= StrictVersion(\"1.0.3\"):\n"
239-
]
240-
},
241-
{
242-
"name": "stdout",
243-
"output_type": "stream",
244-
"text": [
245-
"No hyperparameter file for current model GradientBoosting. Attempting for next model...\n",
246-
"No hyperparameter file for current model RandomForest. Attempting for next model...\n"
247-
]
248-
}
219+
"execution_count": null,
220+
"outputs": [],
221+
"source": [
222+
"check_perf_job = mm.get(f\"/performanceTasks/{perf_job[0].taskId}/performanceJobs\")\n",
223+
"for j in check_perf_job:\n",
224+
" print(f\"{j.model}: {j.state}\")"
249225
],
226+
"metadata": {
227+
"collapsed": false
228+
}
229+
},
230+
{
231+
"cell_type": "code",
232+
"execution_count": null,
233+
"metadata": {},
234+
"outputs": [],
250235
"source": [
251236
"mp.update_kpis(\"HMEQModels\")"
252237
]
@@ -261,28 +246,9 @@
261246
},
262247
{
263248
"cell_type": "code",
264-
"execution_count": 11,
249+
"execution_count": null,
265250
"metadata": {},
266-
"outputs": [
267-
{
268-
"name": "stdout",
269-
"output_type": "stream",
270-
"text": [
271-
"[0.47701149425287354, 0.304921968787515, 0.24622641509433962, 0.21799561082662766]\n",
272-
"Uploading custom kpis to SAS Viya...\n"
273-
]
274-
},
275-
{
276-
"data": {
277-
"text/plain": [
278-
"[<class 'sasctl.core.RestObj'>(headers={}, data={'name': 'TimeSK', 'value': '4', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': 'TimeLabel', 'value': 'Q4', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': 'ProjectUUID', 'value': '7a6dfcc7-8356-4809-92d2-1725569cdab1', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': 'ModelName', 'value': 'DecisionTreeClassifier', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': 'ModelUUID', 'value': '4ff0e60b-4177-487c-b1ca-d78ac4812be4', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': 'ModelFlag', 'value': '-1', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': '_AUC_', 'value': '0.4123305448', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': '_F1_', 'value': '0.344488189', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': '_TPR_', 'value': '1', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': '_FPR_', 'value': '1', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': '_MISC_', 'value': '0.6822235434', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': '_ASE_', 'value': '0.6822235434', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': '_Cumlift5_', 'value': '0.8091228488', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': '_Cumlift10_', 'value': '0.8091228488', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': '_Cumlift15_', 'value': '0.8091228488', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': '_Cumlift20_', 'value': '0.8091228488', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': '_Cumlift30_', 'value': '0.8091228488', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': '_RSquared_', 'value': 'null', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': 'Datetime', 'value': '03Apr2023:19:05:34', 'systemKPIFlag': True}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': '_MICRO_AVG_F1_', 'value': 'null', 'systemKPIFlag': False}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': '_MACRO_AVG_F1_', 'value': 'null', 'systemKPIFlag': False}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': '_WEIGHTED_AVG_F1_', 'value': 'null', 'systemKPIFlag': False}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': '_GLOBAL_ACCURACY_', 'value': 'null', 'systemKPIFlag': False}), <class 'sasctl.core.RestObj'>(headers={}, data={'name': 'jaccard', 'value': '0.2179956108', 'systemKPIFlag': False})]"
279-
]
280-
},
281-
"execution_count": 11,
282-
"metadata": {},
283-
"output_type": "execute_result"
284-
}
285-
],
251+
"outputs": [],
286252
"source": [
287253
"from sklearn.metrics import jaccard_score\n",
288254
"\n",
@@ -306,7 +272,6 @@
306272
"\n",
307273
"print(jaccard_list)\n",
308274
"\n",
309-
"#TODO: allow option to add multiple of same custom KPI\n",
310275
"model = mr.get_model(model_name)\n",
311276
"mm.create_custom_kpi(\n",
312277
" model=model.id,\n",
@@ -327,146 +292,9 @@
327292
},
328293
{
329294
"cell_type": "code",
330-
"execution_count": 12,
295+
"execution_count": null,
331296
"metadata": {},
332-
"outputs": [
333-
{
334-
"name": "stderr",
335-
"output_type": "stream",
336-
"text": [
337-
"~\\sasctl\\pzmm\\model_parameters.py:251: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.\n",
338-
" if pd.__version__ >= StrictVersion(\"1.0.3\"):\n"
339-
]
340-
},
341-
{
342-
"name": "stdout",
343-
"output_type": "stream",
344-
"text": [
345-
"No hyperparameter file for current model GradientBoosting. Attempting for next model...\n",
346-
"No hyperparameter file for current model RandomForest. Attempting for next model...\n",
347-
"[\n",
348-
" {\n",
349-
" \"hyperparameters\": {\n",
350-
" \"ccp_alpha\": 0.0,\n",
351-
" \"class_weight\": null,\n",
352-
" \"criterion\": \"gini\",\n",
353-
" \"max_depth\": null,\n",
354-
" \"max_features\": null,\n",
355-
" \"max_leaf_nodes\": null,\n",
356-
" \"min_impurity_decrease\": 0.0,\n",
357-
" \"min_impurity_split\": null,\n",
358-
" \"min_samples_leaf\": 1,\n",
359-
" \"min_samples_split\": 2,\n",
360-
" \"min_weight_fraction_leaf\": 0.0,\n",
361-
" \"presort\": \"deprecated\",\n",
362-
" \"random_state\": 42,\n",
363-
" \"splitter\": \"best\",\n",
364-
" \"example\": 1\n",
365-
" },\n",
366-
" \"kpis\": {\n",
367-
" \"Q1\": {\n",
368-
" \"TimeSK\": \"1\",\n",
369-
" \"ProjectUUID\": \"7a6dfcc7-8356-4809-92d2-1725569cdab1\",\n",
370-
" \"ModelName\": \"DecisionTreeClassifier\",\n",
371-
" \"ModelFlag\": \"-1\",\n",
372-
" \"_AUC_\": \"0.1803248068\",\n",
373-
" \"_F1_\": \"0.1928552243\",\n",
374-
" \"_TPR_\": \"1\",\n",
375-
" \"_FPR_\": \"1\",\n",
376-
" \"_MISC_\": \"0.9188466112\",\n",
377-
" \"_ASE_\": \"0.9188466112\",\n",
378-
" \"_Cumlift5_\": \"0.3508315027\",\n",
379-
" \"_Cumlift10_\": \"0.3508315027\",\n",
380-
" \"_Cumlift15_\": \"0.3508315027\",\n",
381-
" \"_Cumlift20_\": \"0.3508315027\",\n",
382-
" \"_Cumlift30_\": \"0.3508315027\",\n",
383-
" \"_RSquared_\": null,\n",
384-
" \"Datetime\": \"03Apr2023:19:05:34\",\n",
385-
" \"_MICRO_AVG_F1_\": null,\n",
386-
" \"_MACRO_AVG_F1_\": null,\n",
387-
" \"_WEIGHTED_AVG_F1_\": null,\n",
388-
" \"_GLOBAL_ACCURACY_\": null,\n",
389-
" \"jaccard\": \"0.4770114943\"\n",
390-
" },\n",
391-
" \"Q2\": {\n",
392-
" \"TimeSK\": \"2\",\n",
393-
" \"ProjectUUID\": \"7a6dfcc7-8356-4809-92d2-1725569cdab1\",\n",
394-
" \"ModelName\": \"DecisionTreeClassifier\",\n",
395-
" \"ModelFlag\": \"-1\",\n",
396-
" \"_AUC_\": \"0.2989680477\",\n",
397-
" \"_F1_\": \"0.2524675325\",\n",
398-
" \"_TPR_\": \"1\",\n",
399-
" \"_FPR_\": \"1\",\n",
400-
" \"_MISC_\": \"0.8278834721\",\n",
401-
" \"_ASE_\": \"0.8278834721\",\n",
402-
" \"_Cumlift5_\": \"0.5839659582\",\n",
403-
" \"_Cumlift10_\": \"0.5839659582\",\n",
404-
" \"_Cumlift15_\": \"0.5839659582\",\n",
405-
" \"_Cumlift20_\": \"0.5839659582\",\n",
406-
" \"_Cumlift30_\": \"0.5839659582\",\n",
407-
" \"_RSquared_\": null,\n",
408-
" \"Datetime\": \"03Apr2023:19:05:34\",\n",
409-
" \"_MICRO_AVG_F1_\": null,\n",
410-
" \"_MACRO_AVG_F1_\": null,\n",
411-
" \"_WEIGHTED_AVG_F1_\": null,\n",
412-
" \"_GLOBAL_ACCURACY_\": null,\n",
413-
" \"jaccard\": \"0.3049219688\"\n",
414-
" },\n",
415-
" \"Q3\": {\n",
416-
" \"TimeSK\": \"3\",\n",
417-
" \"ProjectUUID\": \"7a6dfcc7-8356-4809-92d2-1725569cdab1\",\n",
418-
" \"ModelName\": \"DecisionTreeClassifier\",\n",
419-
" \"ModelFlag\": \"-1\",\n",
420-
" \"_AUC_\": \"0.3563914522\",\n",
421-
" \"_F1_\": \"0.2858598726\",\n",
422-
" \"_TPR_\": \"1\",\n",
423-
" \"_FPR_\": \"1\",\n",
424-
" \"_MISC_\": \"0.7624851367\",\n",
425-
" \"_ASE_\": \"0.7624851367\",\n",
426-
" \"_Cumlift5_\": \"0.6941192899\",\n",
427-
" \"_Cumlift10_\": \"0.6941192899\",\n",
428-
" \"_Cumlift15_\": \"0.6941192899\",\n",
429-
" \"_Cumlift20_\": \"0.6941192899\",\n",
430-
" \"_Cumlift30_\": \"0.6941192899\",\n",
431-
" \"_RSquared_\": null,\n",
432-
" \"Datetime\": \"03Apr2023:19:05:34\",\n",
433-
" \"_MICRO_AVG_F1_\": null,\n",
434-
" \"_MACRO_AVG_F1_\": null,\n",
435-
" \"_WEIGHTED_AVG_F1_\": null,\n",
436-
" \"_GLOBAL_ACCURACY_\": null,\n",
437-
" \"jaccard\": \"0.2462264151\"\n",
438-
" },\n",
439-
" \"Q4\": {\n",
440-
" \"TimeSK\": \"4\",\n",
441-
" \"ProjectUUID\": \"7a6dfcc7-8356-4809-92d2-1725569cdab1\",\n",
442-
" \"ModelName\": \"DecisionTreeClassifier\",\n",
443-
" \"ModelFlag\": \"-1\",\n",
444-
" \"_AUC_\": \"0.4123305448\",\n",
445-
" \"_F1_\": \"0.344488189\",\n",
446-
" \"_TPR_\": \"1\",\n",
447-
" \"_FPR_\": \"1\",\n",
448-
" \"_MISC_\": \"0.6822235434\",\n",
449-
" \"_ASE_\": \"0.6822235434\",\n",
450-
" \"_Cumlift5_\": \"0.8091228488\",\n",
451-
" \"_Cumlift10_\": \"0.8091228488\",\n",
452-
" \"_Cumlift15_\": \"0.8091228488\",\n",
453-
" \"_Cumlift20_\": \"0.8091228488\",\n",
454-
" \"_Cumlift30_\": \"0.8091228488\",\n",
455-
" \"_RSquared_\": null,\n",
456-
" \"Datetime\": \"03Apr2023:19:05:34\",\n",
457-
" \"_MICRO_AVG_F1_\": null,\n",
458-
" \"_MACRO_AVG_F1_\": null,\n",
459-
" \"_WEIGHTED_AVG_F1_\": null,\n",
460-
" \"_GLOBAL_ACCURACY_\": null,\n",
461-
" \"jaccard\": \"0.2179956108\"\n",
462-
" }\n",
463-
" }\n",
464-
" },\n",
465-
" \"DecisionTreeClassifierHyperparameters.json\"\n",
466-
"]\n"
467-
]
468-
}
469-
],
297+
"outputs": [],
470298
"source": [
471299
"import json\n",
472300
"\n",
@@ -476,13 +304,6 @@
476304
"\n",
477305
"print(json.dumps(hyperparameters, indent=4))"
478306
]
479-
},
480-
{
481-
"cell_type": "code",
482-
"execution_count": null,
483-
"metadata": {},
484-
"outputs": [],
485-
"source": []
486307
}
487308
],
488309
"metadata": {

0 commit comments

Comments
 (0)