hotfixes and ver update

Author: shubhvjain

codegreen_core/data/entsoe.py

@@ -67,6 +67,10 @@ def _refine_data(options, data1):
     """
     # calculate the duration of the time series by finding the difference between the
     # first and the second index (which is of the type `datatime``) and convert this into minutes
+    #print(data1)
+    if len(data1) == 1:
+        return {"data": None, "refine_logs": ["Only one record cannot be processed"]}
+
     durationMin = (data1.index[1] - data1.index[0]).total_seconds() / 60
     # initializing the log list
     refine_logs = []
@@ -132,12 +136,19 @@ def _entsoe_get_actual_generation(options={"country": "", "start": "", "end": ""
     utc_start = _convert_local_to_utc(options["start"]) 
     utc_end =   _convert_local_to_utc(options["end"]) 
     client1 = entsoePandas(api_key=_get_API_token())
-    data1 = client1.query_generation(
-        options["country"],
-        start = utc_start ,
-        end = utc_end ,
-        psr_type=None,
-    )
+    try :
+        data1 = client1.query_generation(
+            options["country"],
+            start = utc_start ,
+            end = utc_end ,
+            psr_type=None,
+        )
+    except Exception as e:
+        print("Error in fetching data from ENTSOE")
+        return {
+            "data": None,
+            "duration": 0,
+        }
     # drop columns with actual consumption values (we want actual aggregated generation values)
     columns_to_drop = [col for col in data1.columns if col[1] == "Actual Consumption"]
     data1 = data1.drop(columns=columns_to_drop)
@@ -149,9 +160,13 @@ def _entsoe_get_actual_generation(options={"country": "", "start": "", "end": ""
     # refine the dataframe. see the refine method
     data2 = _refine_data(options, data1)
     refined_data = data2["data"]
-    refined_data = refined_data.reset_index(drop=True)
+    
     # finding the duration of the time series data
-    durationMin = (data1.index[1] - data1.index[0]).total_seconds() / 60
+    if(refined_data is not None):
+        refined_data = refined_data.reset_index(drop=True)
+        durationMin = (data1.index[1] - data1.index[0]).total_seconds() / 60
+    else:
+        durationMin = 0 
     return {
         "data": refined_data,
         "duration": durationMin,
@@ -274,15 +289,15 @@ def _format_energy_data(df):
 # the main methods
 
 
-def get_actual_production_percentage(country, start, end, interval60=False) -> dict:
+def get_actual_production_percentage(country, start, end, interval60=True) -> dict:
     """Returns time series data containing the percentage of energy generated from various sources for the specified country within the selected time period.
     It also includes the percentage of energy from renewable and non renewable sources. The data is fetched from the APIs is subsequently refined.
     To obtain data in 60-minute intervals (if not already available), set 'interval60' to True
 
     :param str country: The 2 alphabet country code.
     :param datetime start: The start date for data retrieval. A Datetime object. Note that this date will be rounded to the nearest hour.
     :param datetime end: The end date for data retrieval. A datetime object. This date is also rounded to the nearest hour.
-    :param boolean interval60: To convert the data into 60 min time interval. False by default 
+    :param boolean interval60: To convert the data into 60 min time interval. True by default 
     :return: A DataFrame containing the hourly energy production mix and percentage of energy generated from renewable and non renewable sources.
     :return: A dictionary containing:
       - `error`: A string with an error message, empty if no errors.
@@ -322,6 +337,15 @@ def get_actual_production_percentage(country, start, end, interval60=False) -> d
         # get actual generation data per production type and convert it into 60 min interval if required
         totalRaw = _entsoe_get_actual_generation(options)
         total = totalRaw["data"]
+        
+        if total is None :
+            # no data to process further 
+            return {
+                "data": None,
+                "data_available": False,
+                "error": "Data is not available"
+            }
+
         duration = totalRaw["duration"]
         if options["interval60"] == True and totalRaw["duration"] != 60.0:
             table = _convert_to_60min_interval(totalRaw)

codegreen_core/data/offline.py

@@ -105,24 +105,31 @@ def _sync_offline_file(country):
     current_time = datetime.now()
     # storing data from 5 hours from now.
     end_time =   _round_to_nearest_hour(current_time) - timedelta(hours=5)
-    
+    print(country)
+    print("Checking for file ",json_file_path)
     if not (os.path.exists(json_file_path) and os.path.exists(csv_file_path)):
         print("Files do not exist. Gathering new data.")
         try:
           data = _gather_energy_data(country, start_time, end_time)
-
-          data.to_csv(csv_file_path, index=False)
-          metadata = {
-              "country": country,
-              "first_start_time": int(data.iloc[0]["startTime"].timestamp()),
-              "last_start_time": int(data.iloc[-1]["startTime"].timestamp()),
-              "created_on": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
-              "updated_on": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
-          }
-          with open(json_file_path, "w") as f:
-              json.dump(metadata, f, indent=4)
-          log_stuff("Successfully created new offline file for "+country)
-          return data
+          if data :   
+            data.to_csv(csv_file_path, index=False)
+            first_start_time1 = data.iloc[0]["startTime"]
+            last_start_time1 = data.iloc[-1]["startTime"]
+            metadata = {
+                "country": country,
+                "first_start_time": int(first_start_time1.timestamp()),
+                "last_start_time": int(last_start_time1.timestamp()),
+                "created_on": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                "updated_on": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                "message" : f"Data ranges from {first_start_time1.strftime('%Y-%m-%d %H:%M:%S')} to {last_start_time1.strftime('%Y-%m-%d %H:%M:%S')}"
+
+            }
+            with open(json_file_path, "w") as f:
+                json.dump(metadata, f, indent=4)
+            log_stuff("Successfully created new offline file for "+country)
+            return data
+          else:
+              print("Data not available") 
         except Exception as e:
             print(e)
     else:
@@ -138,23 +145,34 @@ def _sync_offline_file(country):
 
         update_required = False
         if start_diff.total_seconds() > 0:
-            print("Gathering missing data before current start time.")
+            print("Gathering missing data before current start time in the file.")
             new_data = _gather_energy_data(country, start_time, current_start_time )
             df = pd.concat([new_data, df], ignore_index=True)
             update_required = True
         if end_diff.total_seconds() > 0:
-            print("Gathering missing data after current end time.")
-            new_data = _gather_energy_data(country, current_end_time, end_time)
-            #print(new_data)
-            df = pd.concat([df, new_data], ignore_index=True)
-            update_required = True
+            try:
+                print("Gathering missing data after current end time in the file.")
+                new_data = _gather_energy_data(country, current_end_time, end_time)
+                #print(new_data)
+                if new_data is not None : 
+                    df = pd.concat([df, new_data], ignore_index=True)
+                    update_required = True
+                else : 
+                    print("  No new data available")
+            except Exception as e : 
+                print("Error in fetching current data. This is possibly because there is no new data to fetch.")
+                print(e)
+
         if update_required:
           df["startTime"] = pd.to_datetime(df["startTime"])
           df = df.sort_values(by="startTime")
           df.to_csv(csv_file_path, index=False)
-          metadata["first_start_time"] = int(df.iloc[0]["startTime"].timestamp())
-          metadata["last_start_time"] = int(df.iloc[-1]["startTime"].timestamp())
+          first_start_time = df.iloc[0]["startTime"]
+          last_start_time = df.iloc[-1]["startTime"]
+          metadata["first_start_time"] = int(first_start_time.timestamp())
+          metadata["last_start_time"] = int(last_start_time.timestamp())
           metadata["updated_on"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+          metadata["message"] = f"Data ranges from {first_start_time.strftime('%Y-%m-%d %H:%M:%S')} to {last_start_time.strftime('%Y-%m-%d %H:%M:%S')}"
           with open(json_file_path, "w") as f:
             json.dump(metadata, f, indent=4)
           log_stuff("Successfully synced offline file for "+country)

codegreen_core/utilities/config.py

@@ -8,6 +8,7 @@ class ConfigError(Exception):
 
 class Config:
     config_data = None
+    config_file_path = None
     section_name = "codegreen"
     all_keys = [
         {
@@ -116,6 +117,7 @@ def load_config(self, file_path=None):
 
         self.config_data = configparser.ConfigParser()
         self.config_data.read(file_path)
+        self.config_file_path = file_path
 
         if self.section_name not in self.config_data:
             self.config_data[self.section_name] = {}
@@ -164,3 +166,8 @@ def get(self, key):
             print("Config not found")
             print(key)
             raise e
+
+    @classmethod
+    def get_config_file_path(self):
+        """Returns the path of the config file"""
+        return self.config_file_path

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "codegreen_core"
-version = "0.0.5"
+version = "0.0.6"
 description = "This package helps you become aware of the carbon footprint of your computation"
 authors = ["Anne Hartebrodt <anne.hartebrodt@fau.de>","Shubh Vardhan Jain <shubh.v.jain@fau.de>"]
 readme = "README.md"