Merge pull request #934 from green-coding-solutions/Byte-Normalization

Byte normalization - SI Units

api/main.py

@@ -142,7 +142,7 @@ def authenticate(authentication_token=Depends(header_scheme), request: Request =
     parsed_url = urlparse(str(request.url))
     try:
 
-        if not authentication_token: # Note that if no token is supplied this will authenticate as the DEFAULT user, which in FOSS systems has full capabilities
+        if not authentication_token or authentication_token.strip() == '': # Note that if no token is supplied this will authenticate as the DEFAULT user, which in FOSS systems has full capabilities
             authentication_token = 'DEFAULT'
 
         user = User.authenticate(SecureVariable(authentication_token))
@@ -1427,8 +1427,8 @@ class EnergyData(BaseModel):
     machine: UUID
     project: Optional[str] = None
     tags: Optional[str] = None
-    time_stamp: str # is expected to be in microseconds
-    energy_value: str # is expected to be in mJ
+    time_stamp: str # value is in microseconds
+    energy_value: str # value is in Joules
 
     @field_validator('company', 'project', 'tags')
     @classmethod

config.yml.example

@@ -162,6 +162,7 @@ measurement:
 #        CPUCores: 4
 #        CPUThreads: 4
 #        TDP: 65
+######### The value for memory must be in GB not in GiB
 #        HW_MemAmountGB: 16
 #        Hardware_Availability_Year: 2011
   #--- END

frontend/ci.html

@@ -151,7 +151,7 @@ <h3 data-tooltip="The runs table shows all measurements your pipeline has made i
                     <th>Commit Hash</th>
                     <th>Location</th>
                     <th>Grid Intensity</th>
-                    <th>CO2eq of run</th>
+                    <th>gCO2eq of run</th>
                 </tr>
             </thead>
             <tbody id="ci-table"></tbody>

frontend/js/helpers/main.js

@@ -156,7 +156,14 @@ async function makeAPICall(path, values=null, force_authentication_token=null) {
         var options = { method: 'GET', headers: {} }
     }
 
-    options.headers['X-Authentication'] = (force_authentication_token == null) ? localStorage.getItem('authentication_token'): force_authentication_token;
+    if (force_authentication_token != null && force_authentication_token != '') {
+        options.headers['X-Authentication'] = force_authentication_token;
+    } else {
+        const authentication_token = localStorage.getItem('authentication_token');
+        if (force_authentication_token != null && force_authentication_token != '') {
+            options.headers['X-Authentication'] = force_authentication_token;
+        }
+    }
 
     let json_response = null;
     if(localStorage.getItem('remove_idle') == 'true') path += "?remove_idle=true"

lib/system_checks.py

@@ -93,8 +93,8 @@ def check_utf_encoding():
     (check_one_energy_and_scope_machine_provider, Status.ERROR, 'single energy scope machine provider', 'Please only select one provider with energy and scope machine'),
     (check_tmpfs_mount, Status.INFO, 'tmpfs mount', 'We recommend to mount tmp on tmpfs'),
     (check_cpu_utilization, Status.WARN, '< 5% CPU utilization', 'Your system seems to be busy. Utilization is above 5%. Consider terminating some processes for a more stable measurement.'),
-    (check_free_disk, Status.ERROR, '1GB free hdd space', 'We recommend to free up some disk space'),
-    (check_free_memory, Status.ERROR, 'free memory', 'No free memory! Please kill some programs'),
+    (check_free_disk, Status.ERROR, '1 GiB free hdd space', 'We recommend to free up some disk space (< 1GiB available)'),
+    (check_free_memory, Status.ERROR, '1 GiB free memory', 'No free memory! Please kill some programs (< 1GiB available)'),
     (check_docker_daemon, Status.ERROR, 'docker daemon', 'The docker daemon could not be reached. Are you running in rootless mode or have added yourself to the docker group? See installation: [See https://docs.green-coding.io/docs/installation/]'),
     (check_containers_running, Status.WARN, 'running containers', 'You have other containers running on the system. This is usually what you want in local development, but for undisturbed measurements consider going for a measurement cluster [See https://docs.green-coding.io/docs/installation/installation-cluster/].'),
     (check_utf_encoding, Status.ERROR, 'utf file encoding', 'Your system encoding is not set to utf-8. This is needed as we need to parse console output.'),

metric_providers/memory/used/procfs/system/source.c

@@ -48,6 +48,7 @@ static unsigned long long int get_memory_procfs() {
         exit(1);
     }
 
+    // note that here we need to use 1024 instead of 1000 as we are already coming from kiB and not kB
     mem_used = (mem_total - mem_available) * 1024; // outputted value is in Bytes then
 
     fclose(fd);

metric_providers/network/io/cgroup/container/provider.py

@@ -18,6 +18,7 @@ def __init__(self, resolution, rootless=False, skip_check=False):
     def read_metrics(self, run_id, containers=None):
         df = super().read_metrics(run_id, containers)
 
+
         if df.empty:
             return df
 

migrations/2024_05_19_carbondb.sql

@@ -0,0 +1,2 @@
+ALTER TABLE carbondb_energy_data_day DROP CONSTRAINT unique_machine_project_date;
+CREATE UNIQUE INDEX idx_carbondb_unique_entry ON carbondb_energy_data_day(type, company, machine, project, tags, date );

optimization_providers/resources/utilization.py

@@ -13,10 +13,10 @@
 def memory_to_bytes(memory_str):
     """Convert memory string with units (e.g., '50M', '2G') to bytes."""
     unit_multipliers = {
-        'K': 1024,        # Kilobyte
-        'M': 1024**2,     # Megabyte
-        'G': 1024**3,     # Gigabyte
-        'T': 1024**4      # Terabyte
+        'K': 1_000,         # Kilobyte
+        'M': 1_000_000,     # Megabyte
+        'G': 1_000_000_000, # Gigabyte
+        'T': 1_000_000_000, # Terabyte
     }
 
     if isinstance(memory_str, int) or memory_str[-1].isdigit():
@@ -30,15 +30,15 @@ def memory_to_bytes(memory_str):
     raise ValueError(f"Unrecognized memory unit: {unit}")
 
 # pylint: disable=unused-argument
-@register_reporter('container_memory_utilization', Criticality.INFO, REPORTER_NAME, REPORTER_ICON, req_providers =['MemoryTotalCgroupContainerProvider'])
+@register_reporter('container_memory_utilization', Criticality.INFO, REPORTER_NAME, REPORTER_ICON, req_providers =['MemoryUsedCgroupContainerProvider'])
 def container_memory_utilization(self, run, measurements, repo_path, network, notes, phases):
 
     mem = {}
     for s, d in run.get('usage_scenario').get('services').items():
         if x := d.get('deploy', {}).get('resources', {}).get('limits', {}).get('memory', None):
             mem[s] = memory_to_bytes(x)
 
-    for service, measurement_stats in phases.get('data').get('[RUNTIME]').get('memory_total_cgroup_container').get('data').items():
+    for service, measurement_stats in phases.get('data').get('[RUNTIME]').get('memory_used_cgroup_container').get('data').items():
         if not service in mem:
             self.add_optimization(
                 f"You are not using Memory limits definitions on {service}",
@@ -55,8 +55,8 @@ def container_memory_utilization(self, run, measurements, repo_path, network, no
 
         if (actual_mem_max/mem[service]*100) < MIN_MEM_UTIL:
             self.add_optimization(f"Memory utilization is low in {service}", f'''
-                                The service {service} has the memory set to: {mem[service]} bytes but the max
-                                usage was {actual_mem_max} bytes. The mean was {data[first_item].get('mean', None)} bytes.
+                                The service {service} has the memory set to: {mem[service]}bytes but the max
+                                usage was {actual_mem_max}bytes. The mean was {data[first_item].get('mean', None)}bytes.
                                 Which is a usage of {data[first_item].get('mean', 0)/mem[service]*100}%.
                                 Either you should reserve less memory ressources for the container or increase utilization through caching
                                 more data in memory and thus in turn reducing cpu calculations or network traffic if possible.