Skip to content

Commit

Permalink
Merge pull request #29 from sabotack/26-utilization-using-pre-calcula…
Browse files Browse the repository at this point in the history 
…ted-ratios

Add option for using pre calculated ratios
  • Loading branch information
@sabotack
sabotack authored May 13, 2024
2 parents 08d9ad7 + 1de38c1 commit 3187f6b
Show file tree
Hide file tree
Showing 6 changed files with 225 additions and 288 deletions.
85 changes: 35 additions & 50 deletions p6/linear_optimization/optimizer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@

load_dotenv("variables.env")

OPT_MODELS_OUTPUT_DIR = os.getenv("OPT_MODELS_OUTPUT_DIR")
DATA_OUTPUT_DIR = os.getenv("DATA_OUTPUT_DIR")
OPT_MODELS_OUTPUT_DIR = "optimization_models"

# Environment gurobi license variables
options = {
Expand All @@ -24,7 +25,9 @@
}


def runLinearOptimizationModel(model, links, flows, traffic, timestamp, savelp=False):
def runLinearOptimizationModel(
parserArgs, links, flows, traffic, timestamp, savelp=False
):
"""
Runs the linear optimization model to calculate the link utilization and the average link utilization.
Expand All @@ -47,14 +50,19 @@ def runLinearOptimizationModel(model, links, flows, traffic, timestamp, savelp=F
The total link utilization, the average link utilization, and the link utilization for each link.
"""
logger.info("Started running linear optimization model...")
model = parserArgs.model_type

with gp.Env(params=options) as env, gp.Model(env=env) as m:
# Create optimization model based on the input model
m = gp.Model("network_optimization", env=env)

# Decision variables for path ratios for each source-destination pair
path_ratios = m.addVars(
[(sd, pathNum) for sd in flows for pathNum in range(len(flows[sd]))],
[
(sd, flows[sd][pathNum])
for sd in flows
for pathNum in range(len(flows[sd]))
],
vtype=GRB.CONTINUOUS,
name="PathRatios",
)
Expand Down Expand Up @@ -82,11 +90,10 @@ def runLinearOptimizationModel(model, links, flows, traffic, timestamp, savelp=F
# Constraints for each link's utilization
# Consists of the sum of ratios and traffic for each path related to the link
for link in links:
linkTuple = tuple((link[:5], link[5:]))
link_flow = gp.quicksum(
(
path_ratios[sd, pathNum] * traffic[sd]
if linkTuple in zip(flows[sd][pathNum][:-1], flows[sd][pathNum][1:])
path_ratios[sd, flows[sd][pathNum]] * traffic[sd]
if link in flows[sd][pathNum]
else 0
)
for sd in links[link]["listFlows"]
Expand Down Expand Up @@ -118,12 +125,15 @@ def runLinearOptimizationModel(model, links, flows, traffic, timestamp, savelp=F
m.addConstr(path_ratios.sum(sd, "*") == 1, name=f"traffic_split_{sd}")

if savelp:
if not os.path.exists(OPT_MODELS_OUTPUT_DIR):
os.makedirs(OPT_MODELS_OUTPUT_DIR)
dayOutputDir = (
f"{DATA_OUTPUT_DIR}/day{parserArgs.day}/{OPT_MODELS_OUTPUT_DIR}/{model}"
)
if not os.path.exists(dayOutputDir):
os.makedirs(dayOutputDir)

ts = datetime.now().strftime("%Y%m%d")
time = (timestamp[:3] + timestamp[4:-6]).lower()
m.write(f"{OPT_MODELS_OUTPUT_DIR}/{ts}_{model}_{time}.lp")
m.write(f"{dayOutputDir}/{ts}_{time}.lp")

logger.info("Started optimization...")
m.optimize()
Expand All @@ -137,67 +147,42 @@ def runLinearOptimizationModel(model, links, flows, traffic, timestamp, savelp=F
logger.debug(f"Optimal path ratios for {sd}:")
for pathNum in range(len(flows[sd])):
ratioData.append(
[timestamp, sd, pathNum, path_ratios[sd, pathNum].x]
[
timestamp,
sd,
flows[sd][pathNum],
path_ratios[sd, flows[sd][pathNum]].x,
]
)
logger.debug(
f" Path {pathNum}: {path_ratios[sd, pathNum].x * 100} %"
f" Path {pathNum}: {path_ratios[sd, flows[sd][pathNum]].x * 100} %"
)

dataUtils.writeDataToFile(
pd.DataFrame(
ratioData, columns=["timestamp", "flowName", "pathNum", "ratio"]
ratioData, columns=["timestamp", "flowName", "path", "ratio"]
),
model,
"ratioData",
parserArgs,
)

# Calculate average, min and max link utilization
linkData = []
totalLinkUtil = 0
minLinkUtil = 0
maxLinkUtil = 0
# Calculate link utilization
utils = {}

for link in links:
linkTuple = tuple((link[:5], link[5:]))
link_flow = sum(
(
path_ratios[sd, pathNum].x * traffic[sd]
if linkTuple
in zip(flows[sd][pathNum][:-1], flows[sd][pathNum][1:])
path_ratios[sd, flows[sd][pathNum]].x * traffic[sd]
if link in flows[sd][pathNum]
else 0
)
for sd in links[link]["listFlows"]
for pathNum in range(len(flows[sd]))
)
linkData.append(
[
timestamp,
link,
links[link]["capacity"],
link_flow / links[link]["capacity"] * 100,
]
)
totalLinkUtil += link_flow / links[link]["capacity"] * 100

# Update min and max link utilization
if (link_flow / links[link]["capacity"] * 100) < minLinkUtil:
minLinkUtil = link_flow / links[link]["capacity"] * 100
if (link_flow / links[link]["capacity"] * 100) > maxLinkUtil:
maxLinkUtil = link_flow / links[link]["capacity"] * 100

dataUtils.writeDataToFile(
pd.DataFrame(
linkData,
columns=["timestamp", "linkName", "capacity", "utilization (%)"],
),
model,
"linkData",
)

avgLinkUtil = totalLinkUtil / len(links)
logger.info(f"Average link utilization: {avgLinkUtil}% for model {model}")

return avgLinkUtil, minLinkUtil, maxLinkUtil
utils[link] = link_flow / links[link]["capacity"] * 100

return utils
elif m.status == GRB.INFEASIBLE:
logger.error("Model is infeasible")
m.computeIIS()
Expand Down
129 changes: 93 additions & 36 deletions p6/main.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import argparse
import os
import pandas as pd
import statistics as stats
import multiprocessing as mp
Expand All @@ -13,7 +14,7 @@

logger = log.setupCustomLogger(__name__)

DATA_DAY = 2
AVG_CAPACITY = int(os.getenv("AVERAGE_CAPACITY"))


def calcLinkUtil(links):
Expand All @@ -27,52 +28,70 @@ def calcLinkUtil(links):
return util


def process_flows_hour(timestamp, flows, traffic, args, linksCopy):
links = linksCopy
def process_flows_hour(timestamp, flows, traffic, args, links):
logger.info(f"Processing {timestamp} with {len(flows)} flows...")
ratios = None

# Read ratios if specified
if args.use_ratios:
hour = timestamp[4:6]
day, ratioType, date = args.use_ratios
ratios = dataUtils.readRatios(date, ratioType, day, hour)

# Initialize totalTraffic and listFlows for all links
for linkKey in links:
links[linkKey]["totalTraffic"] = 0
links[linkKey]["listFlows"] = []

logger.info(f"Processing {timestamp} with {len(flows)} flows...")
for flow in flows:
routers = nwUtils.getRoutersHashFromFlow(flows[flow])
flowLinks = nwUtils.getFlowLinks(routers, links)

# Update links with traffic, and if link is new, add it to links
for linkKey in flowLinks:
if linkKey in links:
links[linkKey]["totalTraffic"] += (
traffic[flow] * flowLinks[linkKey].trafficRatio
)
else:
links[linkKey] = {
"linkStart": flowLinks[linkKey].linkStart,
"linkEnd": flowLinks[linkKey].linkEnd,
"capacity": flowLinks[linkKey].capacity,
"totalTraffic": traffic[flow] * flowLinks[linkKey].trafficRatio,
# Get all links in the flow
linksFlow = nwUtils.getLinksFromFlow(flows[flow])

identicalPaths = True
if ratios is not None:
for path in flows[flow]:
if (flow, path) not in ratios:
identicalPaths = False
break

# Update totalTraffic and listFlows for each link
for link in linksFlow:
if link not in links:
sd = link.split(";")

links[link] = {
"linkStart": sd[0],
"linkEnd": sd[1],
"capacity": AVG_CAPACITY,
"totalTraffic": 0,
"listFlows": [],
}

# Add this flow to the list of flows for this link
links[linkKey]["listFlows"].append(flow)
totalTraffic = 0
for path in flows[flow]:
if link in path:
if ratios is not None and identicalPaths:
totalTraffic += traffic[flow] * float(ratios[flow, path])
else:
totalTraffic += traffic[flow] * (1 / len(flows[flow]))

links[link]["totalTraffic"] += totalTraffic
links[link]["listFlows"].append(flow)

# Run linear optimization or baseline calculations
if args.model_type == CalcType.BASELINE.value:
linkUtil = calcLinkUtil(links)
return [
timestamp,
min(linkUtil.values()),
max(linkUtil.values()),
stats.mean(linkUtil.values()),
]
else:
avgLinkUtil, minLinkUtil, maxLinkUtil = linOpt.runLinearOptimizationModel(
args.model_type, links, flows, traffic, timestamp, args.save_lp_models
linkUtil = linOpt.runLinearOptimizationModel(
args, links, flows, traffic, timestamp, args.save_lp_models
)
logger.info("LINEAR OPTIMIZATION RETURNED!")
return [timestamp, minLinkUtil, maxLinkUtil, avgLinkUtil]

return [
timestamp,
min(linkUtil.values()),
max(linkUtil.values()),
stats.mean(linkUtil.values()),
]


def main():
Expand All @@ -87,23 +106,61 @@ def main():
],
help="type of calculation to run",
)
parser.add_argument(
"day",
type=int,
nargs="?",
default=2,
help="day number of data to process",
)
parser.add_argument(
"-slpm",
"--save-lp-models",
action="store_true",
help="save linear optimization models",
)
parser.add_argument(
"-ur",
"--use-ratios",
nargs=3,
metavar=("DAY", "TYPE", "DATE"),
help="use existing path ratios for calculations",
)
args = parser.parse_args()

if args.use_ratios:
day, ratioType, date = args.use_ratios
if not day.isdigit():
parser.error("Invalid day number.")
if (
not date.isdigit()
or len(date) != 8
or int(date[4:6]) > 12
or int(date[6:]) > 31
):
parser.error("Invalid date. Please use a date in the format YYYYMMDD.")
if ratioType not in [
CalcType.AVERAGE.value,
CalcType.MAX.value,
CalcType.SQUARED.value,
]:
parser.error(
"Invalid ratio type. Please use 'average', 'max' or 'squared'."
)
if args.model_type != CalcType.BASELINE.value:
parser.error(
"Cannot use existing path ratios with the specified model type."
)

# Set start method to spawn to avoid issues with multiprocessing on Windows
set_start_method("spawn")

startTime = pd.Timestamp.now()
logger.info("Started, model_type: " + str(args.model_type))

flows = dataUtils.readFlows(DATA_DAY)
flows = dataUtils.readFlows(args.day)
links = dataUtils.readLinks()
traffic = dataUtils.readTraffic(DATA_DAY)
traffic = dataUtils.readTraffic(args.day)

with mp.Pool(processes=dataUtils.CPU_THREADS) as pool:
results = pool.starmap(
Expand All @@ -119,11 +176,11 @@ def main():

results.sort()
dataUtils.writeDataToFile(
pd.DataFrame(
data=pd.DataFrame(
results, columns=["timestamp", "min_util", "max_util", "avg_util"]
),
args.model_type,
"overviewData",
parserArgs=args,
outputFile="overviewData",
)

endTime = pd.Timestamp.now()
Expand Down
20 changes: 0 additions & 20 deletions p6/models/network.py
View file

This file was deleted.

Loading

0 comments on commit 3187f6b

Please sign in to comment.