Skip to content

851 emissions energy consumed is zero when tracking tasks #852

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 61 additions & 1 deletion codecarbon/emissions_tracker.py
Original file line number Diff line number Diff line change
Expand Up @@ -506,10 +506,48 @@ def stop_task(self, task_name: str = None) -> EmissionsData:
self._scheduler_monitor_power.stop()

task_name = task_name if task_name else self._active_task

logger.info(
f"STOP_TASK_DEBUG: Before _measure_power_and_energy: "
f"CPU Energy: {self._total_cpu_energy.kWh} kWh, "
f"GPU Energy: {self._total_gpu_energy.kWh} kWh, "
f"RAM Energy: {self._total_ram_energy.kWh} kWh, "
f"Total Energy: {self._total_energy.kWh} kWh"
)
self._measure_power_and_energy()
logger.info(
f"STOP_TASK_DEBUG: After _measure_power_and_energy: "
f"CPU Energy: {self._total_cpu_energy.kWh} kWh, "
f"GPU Energy: {self._total_gpu_energy.kWh} kWh, "
f"RAM Energy: {self._total_ram_energy.kWh} kWh, "
f"Total Energy: {self._total_energy.kWh} kWh"
)

emissions_data = self._prepare_emissions_data()
logger.info(
f"STOP_TASK_DEBUG: emissions_data (totals at task stop): "
f"CPU Energy: {emissions_data.cpu_energy} kWh, "
f"GPU Energy: {emissions_data.gpu_energy} kWh, "
f"RAM Energy: {emissions_data.ram_energy} kWh, "
f"Total Energy: {emissions_data.energy_consumed} kWh"
)
if self._previous_emissions is not None:
logger.info(
f"STOP_TASK_DEBUG: self._previous_emissions (totals at task start): "
f"CPU Energy: {self._previous_emissions.cpu_energy} kWh, "
f"GPU Energy: {self._previous_emissions.gpu_energy} kWh, "
f"RAM Energy: {self._previous_emissions.ram_energy} kWh, "
f"Total Energy: {self._previous_emissions.energy_consumed} kWh"
)

emissions_data_delta = self._compute_emissions_delta(emissions_data)
logger.info(
f"STOP_TASK_DEBUG: emissions_data_delta (final for task): "
f"CPU Energy: {emissions_data_delta.cpu_energy} kWh, "
f"GPU Energy: {emissions_data_delta.gpu_energy} kWh, "
f"RAM Energy: {emissions_data_delta.ram_energy} kWh, "
f"Total Energy: {emissions_data_delta.energy_consumed} kWh"
)

task_duration = Time.from_seconds(
time.perf_counter() - self._tasks[task_name].start_time
Expand Down Expand Up @@ -627,6 +665,11 @@ def _prepare_emissions_data(self) -> EmissionsData:
"""
:delta: If 'True', return only the delta comsumption since the last call.
"""
logger.info(
f"PREPARE_EMISSIONS_DATA_DEBUG: Current total energy values being used: "
f"CPU={self._total_cpu_energy.kWh}, GPU={self._total_gpu_energy.kWh}, "
f"RAM={self._total_ram_energy.kWh}, Total={self._total_energy.kWh}"
)
cloud: CloudMetadata = self._get_cloud_metadata()
duration: Time = Time.from_seconds(time.perf_counter() - self._start_time)

Expand Down Expand Up @@ -688,17 +731,34 @@ def _prepare_emissions_data(self) -> EmissionsData:
return total_emissions

def _compute_emissions_delta(self, total_emissions: EmissionsData) -> EmissionsData:
delta_emissions: EmissionsData = total_emissions
logger.info(
f"COMPUTE_EMISSIONS_DELTA_DEBUG: Input total_emissions: "
f"CPU={total_emissions.cpu_energy}, GPU={total_emissions.gpu_energy}, "
f"RAM={total_emissions.ram_energy}, Total={total_emissions.energy_consumed}"
)
if self._previous_emissions is None:
logger.info("COMPUTE_EMISSIONS_DELTA_DEBUG: self._previous_emissions is None.")
self._previous_emissions = total_emissions
delta_emissions: EmissionsData = total_emissions
else:
logger.info(
f"COMPUTE_EMISSIONS_DELTA_DEBUG: Existing self._previous_emissions: "
f"CPU={self._previous_emissions.cpu_energy}, GPU={self._previous_emissions.gpu_energy}, "
f"RAM={self._previous_emissions.ram_energy}, Total={self._previous_emissions.energy_consumed}"
)
# Create a copy
delta_emissions = dataclasses.replace(total_emissions)
# Compute emissions rate from delta
delta_emissions.compute_delta_emission(self._previous_emissions)
# TODO : find a way to store _previous_emissions only when
# TODO : the API call succeeded
self._previous_emissions = total_emissions

logger.info(
f"COMPUTE_EMISSIONS_DELTA_DEBUG: Returning delta_emissions: "
f"CPU={delta_emissions.cpu_energy}, GPU={delta_emissions.gpu_energy}, "
f"RAM={delta_emissions.ram_energy}, Total={delta_emissions.energy_consumed}"
)
return delta_emissions

@abstractmethod
Expand Down
27 changes: 27 additions & 0 deletions codecarbon/output_methods/emissions_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
from collections import OrderedDict
from dataclasses import dataclass

from codecarbon.external.logger import logger


@dataclass
class EmissionsData:
Expand Down Expand Up @@ -51,10 +53,35 @@ def compute_delta_emission(self, previous_emission):
self.duration = delta_duration
delta_emissions = self.emissions - previous_emission.emissions
self.emissions = delta_emissions

logger.info(
f"EMISSION_DATA_DEBUG: Before CPU delta: "
f"self.cpu_energy={self.cpu_energy}, previous_emission.cpu_energy={previous_emission.cpu_energy}"
)
self.cpu_energy -= previous_emission.cpu_energy
logger.info(f"EMISSION_DATA_DEBUG: After CPU delta: self.cpu_energy={self.cpu_energy}")

logger.info(
f"EMISSION_DATA_DEBUG: Before GPU delta: "
f"self.gpu_energy={self.gpu_energy}, previous_emission.gpu_energy={previous_emission.gpu_energy}"
)
self.gpu_energy -= previous_emission.gpu_energy
logger.info(f"EMISSION_DATA_DEBUG: After GPU delta: self.gpu_energy={self.gpu_energy}")

logger.info(
f"EMISSION_DATA_DEBUG: Before RAM delta: "
f"self.ram_energy={self.ram_energy}, previous_emission.ram_energy={previous_emission.ram_energy}"
)
self.ram_energy -= previous_emission.ram_energy
logger.info(f"EMISSION_DATA_DEBUG: After RAM delta: self.ram_energy={self.ram_energy}")

logger.info(
f"EMISSION_DATA_DEBUG: Before energy_consumed delta: "
f"self.energy_consumed={self.energy_consumed}, previous_emission.energy_consumed={previous_emission.energy_consumed}"
)
self.energy_consumed -= previous_emission.energy_consumed
logger.info(f"EMISSION_DATA_DEBUG: After energy_consumed delta: self.energy_consumed={self.energy_consumed}")

if delta_duration > 0:
# emissions_rate in g/s : delta_emissions in kg.CO2 / delta_duration in s
self.emissions_rate = delta_emissions / delta_duration
Expand Down
110 changes: 110 additions & 0 deletions examples/task_zero_energy_debug.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
import os
import time

from codecarbon import EmissionsTracker

# Ensure you have your CODECARBON_API_TOKEN if you use the API
# or configure offline mode as needed.
# For simplicity, this script assumes offline mode if no API key is found,
# but you should adjust it to your actual CodeCarbon setup.

API_KEY = os.environ.get("CODECARBON_API_TOKEN")
OFFLINE_MODE = API_KEY is None

if OFFLINE_MODE:
print("Running in OFFLINE mode as CODECARBON_API_TOKEN is not set.")
# For offline, you might need to specify country_iso_code, etc.
# Adjust these parameters as per your setup or config file
tracker = EmissionsTracker(
project_name="ZeroEnergyTestLoop",
measure_power_secs=1, # Or your desired interval
tracking_mode="process",
# country_iso_code="USA", # Example for offline
# region="california", # Example for offline
log_level="debug", # Set to debug to get all codecarbon logs + our new ones
)
else:
print(
f"Running in ONLINE mode with API Key: {API_KEY[:5]}..."
) # Print part of key for confirmation
tracker = EmissionsTracker(
project_name="ZeroEnergyTestLoop",
measure_power_secs=1, # Or your desired interval
tracking_mode="process",
api_key=API_KEY,
log_level="debug", # Set to debug to get all codecarbon logs + our new ones
)


def busy_task(duration_secs=4):
print(f" Task: Starting busy work for ~{duration_secs} seconds...")
start_time = time.perf_counter()
while time.perf_counter() - start_time < duration_secs:
# Simulate some CPU work
# for _ in range(100000): # Adjust complexity as needed
# pass
time.sleep(2)
end_time = time.perf_counter()
print(f" Task: Finished busy work in {end_time - start_time:.2f} seconds.")


max_rounds = 100 # Safety break for the loop

print("Starting tracking loop. Will stop if energy_consumed is 0.0 for a task.")

try:
for current_round in range(max_rounds):
print(f"Round {current_round + 1}:")
task_name = f"round_{current_round + 1}_task"

tracker.start_task(task_name)
print(f" Tracker: Started task '{task_name}'")

busy_task(duration_secs=4) # Simulate work for about 4 seconds

emissions_data = tracker.stop_task()
print(f" Tracker: Stopped task '{task_name}'")

if emissions_data:
print(f" EmissionsData for {task_name}:")
print(f" Duration: {emissions_data.duration:.4f}s")
print(f" CPU Energy: {emissions_data.cpu_energy:.6f} kWh")
print(f" GPU Energy: {emissions_data.gpu_energy:.6f} kWh")
print(f" RAM Energy: {emissions_data.ram_energy:.6f} kWh")
print(
f" Total Energy Consumed: {emissions_data.energy_consumed:.6f} kWh"
)
print(f" Emissions: {emissions_data.emissions:.6f} kg CO2eq")

if emissions_data.energy_consumed == 0.0:
print("###########################################################")
print(
f"INFO: energy_consumed is 0.0 in round {current_round + 1}. Stopping loop."
)
print("###########################################################")
break
else:
print(f" WARNING: tracker.stop_task() returned None for {task_name}")

# Small pause between rounds, can be adjusted or removed
time.sleep(1)

else: # Executed if the loop completes without break
print(
f"Loop completed {max_rounds} rounds without encountering zero energy consumption."
)

except Exception as e:
print(f"An error occurred: {e}")
finally:
print("Stopping global tracker (if it was started implicitly or if needed).")
# tracker.stop() # Call stop() if you started the main tracker with tracker.start() explicitly.
# For tasks, usually stop_task is enough unless you have a global start().
# If the tracker is only used for tasks like this, explicit stop() might not be needed
# or could even cause issues if not started.
# However, it's good practice if you use the tracker in a broader scope.
# For this script, since we initialize tracker but don't call tracker.start(),
# calling tracker.stop() might error if it wasn't implicitly started.
# The EmissionsTracker's __exit__ method (if used with `with`) handles stop.
# If running tasks like this, the library should handle shutdown on exit or via stop_task effects.
print("Script finished.")
Loading