11"""Command Extraction and Formatting or SSoT Based Jobs."""
22
33import json
4+ from json .decoder import JSONDecodeError
45import logging
56
67from django .template import engines
@@ -38,80 +39,107 @@ def get_django_env():
3839 return jinja_env
3940
4041
42+ def process_empty_result (iterable_type ):
43+ """Helper to map iterable_type on an empty result."""
44+ iterable_mapping = {
45+ "dict" : {},
46+ "str" : "" ,
47+ }
48+ return iterable_mapping .get (iterable_type , [])
49+
50+
51+ def normalize_processed_data (processed_data , iterable_type ):
52+ """Helper to normalize the processed data returned from jdiff/jmespath."""
53+ # If processed_data is an empty data structure, return default based on iterable_type
54+ if not processed_data :
55+ return process_empty_result (iterable_type )
56+ if isinstance (processed_data , str ):
57+ try :
58+ # If processed_data is a json string try to load it into a python datatype.
59+ post_processed_data = json .loads (processed_data )
60+ except (JSONDecodeError , TypeError ):
61+ post_processed_data = processed_data
62+ else :
63+ post_processed_data = processed_data
64+ if isinstance (post_processed_data , list ) and len (post_processed_data ) == 1 :
65+ if isinstance (post_processed_data [0 ], str ):
66+ post_processed_data = post_processed_data [0 ]
67+ else :
68+ if isinstance (post_processed_data [0 ], dict ):
69+ if iterable_type :
70+ if iterable_type == "dict" :
71+ post_processed_data = post_processed_data [0 ]
72+ else :
73+ post_processed_data = post_processed_data [0 ]
74+ return post_processed_data
75+
76+
4177def extract_and_post_process (parsed_command_output , yaml_command_element , j2_data_context , iter_type , job_debug ):
4278 """Helper to extract and apply post_processing on a single element."""
4379 logger = logger = setup_logger ("DEVICE_ONBOARDING_ETL_LOGGER" , job_debug )
4480 # if parsed_command_output is an empty data structure, no need to go through all the processing.
45- if parsed_command_output :
46- j2_env = get_django_env ()
47- jpath_template = j2_env .from_string (yaml_command_element ["jpath" ])
48- j2_rendered_jpath = jpath_template .render (** j2_data_context )
49- logger .debug ("Post Rendered Jpath: %s" , j2_rendered_jpath )
81+ if not parsed_command_output :
82+ return parsed_command_output , normalize_processed_data (parsed_command_output , iter_type )
83+ j2_env = get_django_env ()
84+ # This just renders the jpath itself if any interpolation is needed.
85+ jpath_template = j2_env .from_string (yaml_command_element ["jpath" ])
86+ j2_rendered_jpath = jpath_template .render (** j2_data_context )
87+ logger .debug ("Post Rendered Jpath: %s" , j2_rendered_jpath )
88+ try :
5089 if isinstance (parsed_command_output , str ):
51- parsed_command_output = json .loads (parsed_command_output )
52- try :
53- extracted_value = extract_data_from_json (parsed_command_output , j2_rendered_jpath )
54- except TypeError as err :
55- logger .debug ("Error occurred during extraction: %s" , err )
56- extracted_value = []
57- pre_processed_extracted = extracted_value
58- if yaml_command_element .get ("post_processor" ):
59- # j2 context data changes obj(hostname) -> extracted_value for post_processor
60- j2_data_context ["obj" ] = extracted_value
61- template = j2_env .from_string (yaml_command_element ["post_processor" ])
62- extracted_processed = template .render (** j2_data_context )
63- else :
64- extracted_processed = extracted_value
65- if isinstance (extracted_processed , str ):
6690 try :
67- post_processed_data = json .loads (extracted_processed )
68- except Exception :
69- post_processed_data = extracted_processed
70- else :
71- post_processed_data = extracted_processed
72- if isinstance (post_processed_data , list ) and len (post_processed_data ) == 0 :
73- # means result was empty, change empty result to iterater_type if applicable.
74- if iter_type :
75- if iter_type == "dict" :
76- post_processed_data = {}
77- if iter_type == "str" :
78- post_processed_data = ""
79- if isinstance (post_processed_data , list ) and len (post_processed_data ) == 1 :
80- if isinstance (post_processed_data [0 ], str ):
81- post_processed_data = post_processed_data [0 ]
82- else :
83- if isinstance (post_processed_data [0 ], dict ):
84- if iter_type :
85- if iter_type == "dict" :
86- post_processed_data = post_processed_data [0 ]
87- else :
88- post_processed_data = post_processed_data [0 ]
89- logger .debug ("Pre Processed Extracted: %s" , pre_processed_extracted )
90- logger .debug ("Post Processed Data: %s" , post_processed_data )
91- return pre_processed_extracted , post_processed_data
92- if iter_type :
93- if iter_type == "dict" :
94- post_processed_data = {}
95- if iter_type == "str" :
96- post_processed_data = ""
91+ parsed_command_output = json .loads (parsed_command_output )
92+ except (JSONDecodeError , TypeError ):
93+ logger .debug ("Parsed Command Output is a string but not jsonable: %s" , parsed_command_output )
94+ extracted_value = extract_data_from_json (parsed_command_output , j2_rendered_jpath )
95+ except TypeError as err :
96+ logger .debug ("Error occurred during extraction: %s setting default extracted value to []" , err )
97+ extracted_value = []
98+ pre_processed_extracted = extracted_value
99+ if yaml_command_element .get ("post_processor" ):
100+ # j2 context data changes obj(hostname) -> extracted_value for post_processor
101+ j2_data_context ["obj" ] = extracted_value
102+ template = j2_env .from_string (yaml_command_element ["post_processor" ])
103+ extracted_processed = template .render (** j2_data_context )
97104 else :
98- post_processed_data = []
99- logger .debug ("Pre Processed Extracted: %s" , parsed_command_output )
105+ extracted_processed = extracted_value
106+ post_processed_data = normalize_processed_data (extracted_processed , iter_type )
107+ logger .debug ("Pre Processed Extracted: %s" , pre_processed_extracted )
100108 logger .debug ("Post Processed Data: %s" , post_processed_data )
101- return parsed_command_output , post_processed_data
109+ return pre_processed_extracted , post_processed_data
102110
103111
104112def perform_data_extraction (host , command_info_dict , command_outputs_dict , job_debug ):
105113 """Extract, process data."""
106114 result_dict = {}
107115 sync_vlans = host .defaults .data .get ("sync_vlans" , False )
108116 sync_vrfs = host .defaults .data .get ("sync_vrfs" , False )
117+ get_context_from_pre_processor = {}
118+ if command_info_dict .get ("pre_processor" ):
119+ for pre_processor_name , field_data in command_info_dict ["pre_processor" ].items ():
120+ if isinstance (field_data ["commands" ], dict ):
121+ # only one command is specified as a dict force it to a list.
122+ loop_commands = [field_data ["commands" ]]
123+ else :
124+ loop_commands = field_data ["commands" ]
125+ for show_command_dict in loop_commands :
126+ final_iterable_type = show_command_dict .get ("iterable_type" )
127+ _ , current_field_post = extract_and_post_process (
128+ command_outputs_dict [show_command_dict ["command" ]],
129+ show_command_dict ,
130+ {"obj" : host .name , "original_host" : host .name },
131+ final_iterable_type ,
132+ job_debug ,
133+ )
134+ get_context_from_pre_processor [pre_processor_name ] = current_field_post
109135 for ssot_field , field_data in command_info_dict .items ():
110136 if not sync_vlans and ssot_field in ["interfaces__tagged_vlans" , "interfaces__untagged_vlan" ]:
111137 continue
112138 # If syncing vrfs isn't inscope remove the unneeded commands.
113139 if not sync_vrfs and ssot_field == "interfaces__vrf" :
114140 continue
141+ if ssot_field == "pre_processor" :
142+ continue
115143 if isinstance (field_data ["commands" ], dict ):
116144 # only one command is specified as a dict force it to a list.
117145 loop_commands = [field_data ["commands" ]]
@@ -120,10 +148,12 @@ def perform_data_extraction(host, command_info_dict, command_outputs_dict, job_d
120148 for show_command_dict in loop_commands :
121149 final_iterable_type = show_command_dict .get ("iterable_type" )
122150 if field_data .get ("root_key" ):
151+ original_context = {"obj" : host .name , "original_host" : host .name }
152+ merged_context = {** original_context , ** get_context_from_pre_processor }
123153 root_key_pre , root_key_post = extract_and_post_process (
124154 command_outputs_dict [show_command_dict ["command" ]],
125155 show_command_dict ,
126- { "obj" : host . name , "original_host" : host . name } ,
156+ merged_context ,
127157 final_iterable_type ,
128158 job_debug ,
129159 )
@@ -139,19 +169,23 @@ def perform_data_extraction(host, command_info_dict, command_outputs_dict, job_d
139169 # a list of data that we want to become our nested key. E.g. current_key "Ethernet1/1"
140170 # These get passed into the render context for the template render to allow nested jpaths to use
141171 # the current_key context for more flexible jpath queries.
172+ original_context = {"current_key" : current_key , "obj" : host .name , "original_host" : host .name }
173+ merged_context = {** original_context , ** get_context_from_pre_processor }
142174 _ , current_key_post = extract_and_post_process (
143175 command_outputs_dict [show_command_dict ["command" ]],
144176 show_command_dict ,
145- { "current_key" : current_key , "obj" : host . name , "original_host" : host . name } ,
177+ merged_context ,
146178 final_iterable_type ,
147179 job_debug ,
148180 )
149181 result_dict [field_nesting [0 ]][current_key ][field_nesting [1 ]] = current_key_post
150182 else :
183+ original_context = {"obj" : host .name , "original_host" : host .name }
184+ merged_context = {** original_context , ** get_context_from_pre_processor }
151185 _ , current_field_post = extract_and_post_process (
152186 command_outputs_dict [show_command_dict ["command" ]],
153187 show_command_dict ,
154- { "obj" : host . name , "original_host" : host . name } ,
188+ merged_context ,
155189 final_iterable_type ,
156190 job_debug ,
157191 )
0 commit comments