Skip to content
Snippets Groups Projects
Commit 5a922223 authored by tlee's avatar tlee
Browse files

Thesis branch commit

parent d2da2cd8
No related branches found
No related tags found
No related merge requests found
#Model variables
MODE=train
MODEL_FILEPATH=models/nmea_model.json
MODEL_FILEPATH=models/nmea_model_live.json
NS_FREQUENCY_RANGE = 1000000 #Should be a variable set by user during training/testing is in nanoseconds (1000000 = 0.001 seconds)
ROC_ERROR_THRESHOLD = 1.1 #Variable to dictate the error threshold for the rate of change
#Minio variables
MINIO_ENDPOINT=minio-dev.esg.uwaterloo.ca
MINIO_ACCESS_KEY=
MINIO_SECRET_KEY=
TENANT_ID=
MINIO_ACCESS_KEY=JEogRTx9HiWc5y8ypwd5T
MINIO_SECRET_KEY=sp7CrozKE8A34b6kQ2dh4
TENANT_ID=1636c3c1-3dd8-457d-b4cd-9edc51150ff1
SENSOR_FILEPATH=NMEA-Sensor/can/
\ No newline at end of file
#Model variables
MODE=train
MODEL_FILEPATH=models/peterbilt_door_model.json
NS_FREQUENCY_RANGE = 1000000 #Should be a variable set by user during training/testing is in nanoseconds (1000000 = 0.001 seconds)
ROC_ERROR_THRESHOLD = 1.1 #Variable to dictate the error threshold for the rate of change
#Minio variables
MINIO_ENDPOINT=minio-eet.esg.uwaterloo.ca
MINIO_ACCESS_KEY=HsRVRq7XS73QE6UBz8Xwn
MINIO_SECRET_KEY=VnCUUuQFcjqJt5JKYcg7N
TENANT_ID=0d830ae9-1046-4370-a344-99ff57f85cce
SENSOR_FILEPATH=TRUCK-1-Door-Idle/can/2023-10-06
\ No newline at end of file
#Model variables
MODE=train
MODEL_FILEPATH=models/peterbilt_door_model.json
NS_FREQUENCY_RANGE = 1000000 #Should be a variable set by user during training/testing is in nanoseconds (1000000 = 0.001 seconds)
ROC_ERROR_THRESHOLD = 1.1 #Variable to dictate the error threshold for the rate of change
#Minio variables
MINIO_ENDPOINT=minio-eet.esg.uwaterloo.ca
MINIO_ACCESS_KEY=HsRVRq7XS73QE6UBz8Xwn
MINIO_SECRET_KEY=VnCUUuQFcjqJt5JKYcg7N
TENANT_ID=0d830ae9-1046-4370-a344-99ff57f85cce
SENSOR_FILEPATH=TRUCK-1/can/
\ No newline at end of file
#Model variables
MODE=train
MODEL_FILEPATH=models/nmea_model.json
NS_FREQUENCY_RANGE = 1000000 #Should be a variable set by user during training/testing is in nanoseconds (1000000 = 0.001 seconds)
ROC_ERROR_THRESHOLD = 1.1 #Variable to dictate the error threshold for the rate of change
#Minio variables
MINIO_ENDPOINT=minio-dev.esg.uwaterloo.ca
MINIO_ACCESS_KEY=JEogRTx9HiWc5y8ypwd5T
MINIO_SECRET_KEY=sp7CrozKE8A34b6kQ2dh4
TENANT_ID=1636c3c1-3dd8-457d-b4cd-9edc51150ff1
SENSOR_FILEPATH=NMEA-Sensor-2023-10-19/can/2023-10-19
\ No newline at end of file
#Model variables
MODE=train
MODEL_FILEPATH=models/nmea_model_live.json
NS_FREQUENCY_RANGE = 1000000 #Should be a variable set by user during training/testing is in nanoseconds (1000000 = 0.001 seconds)
ROC_ERROR_THRESHOLD = 1.1 #Variable to dictate the error threshold for the rate of change
#Minio variables
MINIO_ENDPOINT=minio-dev.esg.uwaterloo.ca
MINIO_ACCESS_KEY=JEogRTx9HiWc5y8ypwd5T
MINIO_SECRET_KEY=sp7CrozKE8A34b6kQ2dh4
TENANT_ID=1636c3c1-3dd8-457d-b4cd-9edc51150ff1
SENSOR_FILEPATH=NMEA-Sensor/can/
\ No newline at end of file
#Model variables
MODE=train
MODEL_FILEPATH=models/peterbilt_obdii_model.json
NS_FREQUENCY_RANGE = 1000000 #Should be a variable set by user during training/testing is in nanoseconds (1000000 = 0.001 seconds)
ROC_ERROR_THRESHOLD = 1.1 #Variable to dictate the error threshold for the rate of change
#Minio variables
MINIO_ENDPOINT=minio-eet.esg.uwaterloo.ca
MINIO_ACCESS_KEY=HsRVRq7XS73QE6UBz8Xwn
MINIO_SECRET_KEY=VnCUUuQFcjqJt5JKYcg7N
TENANT_ID=0d830ae9-1046-4370-a344-99ff57f85cce
SENSOR_FILEPATH=TRUCK-2-OBDII-Idle/can/2023-10-06
\ No newline at end of file
#Model variables
MODE=train
MODEL_FILEPATH=models/peterbilt_obdii_model_live.json
NS_FREQUENCY_RANGE = 1000000 #Should be a variable set by user during training/testing is in nanoseconds (1000000 = 0.001 seconds)
ROC_ERROR_THRESHOLD = 1.1 #Variable to dictate the error threshold for the rate of change
#Minio variables
MINIO_ENDPOINT=minio-eet.esg.uwaterloo.ca
MINIO_ACCESS_KEY=HsRVRq7XS73QE6UBz8Xwn
MINIO_SECRET_KEY=VnCUUuQFcjqJt5JKYcg7N
TENANT_ID=0d830ae9-1046-4370-a344-99ff57f85cce
SENSOR_FILEPATH=TRUCK-2/can/
\ No newline at end of file
.env-old 0 → 100644
#Model variables
MODE=train
MODEL_FILEPATH=models/nmea_model_live.json
NS_FREQUENCY_RANGE = 1000000 #Should be a variable set by user during training/testing is in nanoseconds (1000000 = 0.001 seconds)
ROC_ERROR_THRESHOLD = 1.1 #Variable to dictate the error threshold for the rate of change
#Minio variables
MINIO_ENDPOINT=minio-dev.esg.uwaterloo.ca
MINIO_ACCESS_KEY=JEogRTx9HiWc5y8ypwd5T
MINIO_SECRET_KEY=sp7CrozKE8A34b6kQ2dh4
TENANT_ID=1636c3c1-3dd8-457d-b4cd-9edc51150ff1
SENSOR_FILEPATH=NMEA-Sensor/can/
#MINIO_ENDPOINT=minio-eet.esg.uwaterloo.ca
#MINIO_ACCESS_KEY=HsRVRq7XS73QE6UBz8Xwn
#MINIO_SECRET_KEY=VnCUUuQFcjqJt5JKYcg7N
#TENANT_ID=0d830ae9-1046-4370-a344-99ff57f85cce
#SENSOR_FILEPATH=TRUCK-2/can/2023-03-22
#client = Minio("minio-eet.esg.uwaterloo.ca", "readonly_all", "7iK!jUY8wm#j6k#B")
#bucket = "tenant-0d830ae9-1046-4370-a344-99ff57f85cce"
#filepath = "TRUCK-2/can/" TRUCK-2/can/2023-03-22
#compute3.esg.uwaterloo.ca:10003
#minio_remote_user: yLHTdHfqib56qh
#minio_remote_pass: mVR6zxGxDYhGHX
#vpn sudo password qZQUE7NjnVVgcW2bh3
Check edgehub metrics for timing?
Check how long decoding process works (should be around 1.5/2)
Check timestamp recieved in analyzer in real time
Create a .csv when a message from the interceptor is recieved (68 frames per packet) four timestamps: arrival from interceptor, first timestamp,
last timestamp, processing time for packet i.e from receieving to upload to minio
On analyzer get current time and see maybe run as docker container on the local machine?
1.run analyzer off machine
2.run analyzer on machine with local minio instance
Worst case scenario uplaod every 68: and messages with quick can bus. keep adding interceptors until the upload rate for the truck obd port lowers. This is when the message send rate cannot keep up with the can bus load
\ No newline at end of file
......@@ -167,8 +167,8 @@ def test_model(row):
model_update.update_max_spn_value(spnvalue['SPNName'], spnvalue['DecodedSPNValue'], row)
if spnrules_dict.get('MinValue') is None or spnvalue['DecodedSPNValue'] < spnrules_dict['MinValue']:
model_update.update_min_spn_value(spnvalue['SPNName'], spnvalue['DecodedSPNValue'], row)
if spnrules_dict.get('States') is None or str(spnvalue['StateValue']) not in spnrules_dict['States']:
model_update.update_spn_state(spnvalue['SPNName'], spnvalue['StateValue'], row)
#if spnrules_dict.get('States') is None or str(spnvalue['StateValue']) not in spnrules_dict['States']:
# model_update.update_spn_state(spnvalue['SPNName'], spnvalue['StateValue'], row)
if spnrules_dict.get('Unit') is None or spnvalue['SPNUnit'] != spnrules_dict['Unit']:
model_update.update_spn_unit(spnvalue['SPNName'], spnvalue['SPNUnit'], row)
if spnrules_dict.get('Availability') is None or spnvalue['Available'] != spnrules_dict['Availability']:
......
import os
import json
import time
import csv
import pandas as pd
from minio import Minio
......@@ -35,21 +36,33 @@ def check_model():
outfile.write(model_json)
def minio_live_data_listener():
csv_file = 'output.csv' # Name of the CSV file
headers = ['Interceptor Start', 'Interceptor End', 'Processing Start', 'Processing End']
with client.listen_bucket_notification(
bucket,
prefix=filepath,
events=["s3:ObjectCreated:Put"],
) as events:
for event in events:
data_received = time.time_ns()
data_file_path = event["Records"][0]["s3"]["object"]["key"]
client.fget_object(bucket,data_file_path,data_file_path)
data = pd.read_csv(data_file_path, compression='gzip')
before_data = time.time_ns()
data_processing.process_data(data)
after_data = time.time_ns()
data_complete = time.time_ns()
#Prints out stats on how long data processing took
row = [before_data, int(data.head(1)['timestamp']), int(data.tail(1)['timestamp']), after_data - before_data]
print(row)
row = [int(data.head(1)['timestamp']), int(data.tail(1)['timestamp']), data_received, data_complete]
#print(row)
# Check if CSV exists and write row
file_exists = os.path.isfile(csv_file)
with open(csv_file, 'a', newline='') as f:
writer = csv.writer(f)
if not file_exists:
writer.writerow(headers) # Writing headers if file does not exist
writer.writerow(row) # Writing data
if os.path.exists(data_file_path):
os.remove(data_file_path)
......@@ -99,5 +112,5 @@ if __name__ == "__main__":
#run_app()
##Unstable runs grafana plus decoder
#minio_live_data_listener()
minio_stored_data_listener() #This is for using stored minio data instead of live
minio_live_data_listener()
#minio_stored_data_listener() #This is for using stored minio data instead of live
import json
from collections import defaultdict
def load_json_model(model_path):
try:
with open(model_path, 'r') as file:
data = json.load(file)
return data
except Exception as e:
print(f"Error loading model from path '{model_path}': {e}")
return None
# List of model paths
model_paths = [
'models/nmea_model.json',
'models/peterbilt_door_model.json',
'models/peterbilt_obdii_model.json'
]
comparison_data = {
"door": {"decoded": set(), "undecoded": set()},
"obdii": {"decoded": set(), "undecoded": set()}
}
shared_data = {
"decoded": set(),
"undecoded": set(),
"sas": defaultdict(dict) # Changed to dict to store frequencies
}
# Load each model and print the stats
for path in model_paths:
model_data = load_json_model(path)
if model_data is not None:
model_type = "door" if "door" in path else "obdii" if "obdii" in path else None
if model_type:
decoded_pgns = set()
undecoded_pgns = set()
for pgn, content in model_data.items():
decoded = any(details.get('SPNRules') for details in content.values())
if decoded:
decoded_pgns.add(pgn)
else:
undecoded_pgns.add(pgn)
if model_type in ["door", "obdii"]:
for sa, details in content.items():
shared_data["sas"][pgn][sa] = details.get('Frequencies', [])
comparison_data[model_type]["decoded"] = decoded_pgns
comparison_data[model_type]["undecoded"] = undecoded_pgns
# Print the shared data and frequencies
print("Shared PGNs and Frequencies:")
for pgn, sas in shared_data["sas"].items():
is_decoded = pgn in shared_data["decoded"]
status = "Decoded" if is_decoded else "Undecoded" if pgn in shared_data["undecoded"] else "Not shared"
print(f"PGN {pgn} ({status}):")
for sa, freq in sas.items():
print(f" SA {sa}: Frequencies {freq}")
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment