qwphr/ess-moxa-configuration-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[WiP] Upgrade for telemetry history

Browse Source
This commit is contained in:
Quentin WEPHRE
2025-10-06 08:04:06 +02:00
parent a910e0ba31
commit 013d9c820f
2 changed files with 62 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -17,3 +17,4 @@ Python/build/*
Python/dist/* Python/dist/*
*.spec *.spec
dist/* dist/*
*.7z

123
Python/cube_ssh_get_space.py
View File

@@ -21,8 +21,8 @@ def resource_path(relative_path):
dotenv_path = resource_path('.env') dotenv_path = resource_path('.env')
load_dotenv(dotenv_path=dotenv_path) load_dotenv(dotenv_path=dotenv_path)
ip_address_prefix = "10.84.171." # Myrtle subnet ip_address_prefix = "10.84.165." # Danish subnet
ip_address_range = range(3, 5) # From 1 to 57 ip_address_range = range(131, 132) # From 131 to 187
ENV_SSH = { ENV_SSH = {
"DEFAULT_CUBE_LINUX_ADMIN_USER": os.getenv("DEFAULT_CUBE_LINUX_ADMIN_USER"), "DEFAULT_CUBE_LINUX_ADMIN_USER": os.getenv("DEFAULT_CUBE_LINUX_ADMIN_USER"),
@@ -32,37 +32,46 @@ ENV_SSH = {
ssh_username = ENV_SSH["DEFAULT_CUBE_LINUX_ADMIN_USER"] ssh_username = ENV_SSH["DEFAULT_CUBE_LINUX_ADMIN_USER"]
ssh_password = ENV_SSH["DEFAULT_CUBE_LINUX_ADMIN_PASSWORD"] ssh_password = ENV_SSH["DEFAULT_CUBE_LINUX_ADMIN_PASSWORD"]
def execute_ssh_command(ip, command): def execute_ssh_command(ip, command, client):
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try: try:
client.connect(ip, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False) stdin, stdout, stderr = client.exec_command(command, timeout=180)
stdin, stdout, stderr = client.exec_command(command) exit_status = stdout.channel.recv_exit_status()
result = stdout.read().decode().lower().strip() result = stdout.read().decode().lower().strip()
error = stderr.read().decode('utf-8')
if exit_status == 0:
print(f"✅")
else:
print(f"❌")
print(f"{error}")
raise Exception(f"{str(error)}")
return result return result
except Exception as e: except Exception as e:
print(f"SSH Error: {str(e)}", flush=True) print(f"SSH error: {str(e)} --- {str(error)}", flush=True)
raise raise
finally: finally:
client.close() client.close()
def execute_sudo_ssh_command(ip, command): def execute_sudo_ssh_command(ip, command, client):
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try: try:
client.connect(ip, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False)
client.get_transport().set_keepalive(10)
quoted_command = f"bash -c {shlex.quote(command)}" quoted_command = f"bash -c {shlex.quote(command)}"
sudo_command = f"sudo -S -p '' {quoted_command}" sudo_command = f"sudo -S -p '' {quoted_command}"
stdin, stdout, stderr = client.exec_command(sudo_command, timeout=60) stdin, stdout, stderr = client.exec_command(sudo_command, timeout=180)
time.sleep(3) time.sleep(3)
stdin.write(ssh_password + '\n') stdin.write(ssh_password + '\n')
stdin.flush() stdin.flush()
exit_status = stdout.channel.recv_exit_status()
error = stderr.read().decode().strip() output = stdout.read().decode('utf-8')
if error: error = stderr.read().decode('utf-8')
raise Exception(f"sudo SSH command failed with error: {error}") if exit_status == 0:
print(f"{output}")
else:
print(f"❌")
print(f"{error}")
raise Exception("Error during SSH sudo command.")
result = stdout.read().decode().lower().strip() result = stdout.read().decode().lower().strip()
return result return result
@@ -90,9 +99,11 @@ def scp_get_file(ip, remote_path, local_path):
username=ssh_username, username=ssh_username,
password=ssh_password, password=ssh_password,
allow_agent=False, allow_agent=False,
look_for_keys=False look_for_keys=False,
timeout=180
) )
client.get_transport().set_keepalive(5)
with SCPClient(client.get_transport()) as scp: with SCPClient(client.get_transport()) as scp:
scp.get(remote_path, local_path) scp.get(remote_path, local_path)
@@ -131,14 +142,21 @@ def main():
continue continue
print(f"Getting small datasets:", flush=True) print(f"Getting small datasets:", flush=True)
metric_names = [] #["ModTmpAvg", "ModTmpMin", "ModTmpMax", "StrModTmpAvg"] metric_names = ["ModTmpAvg", "ModTmpMin", "ModTmpMax", "StrModTmpAvg", "NStrCon", "DCDischEnergy", "DCChEnergy", "SoC", "StrSoC"]
start_date = "2024-01-01" start_date = "2024-01-01"
end_date = "2026-01-01" end_date = "2026-01-01"
activate_ssh(ip_address)
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(ip_address, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False, timeout=180, banner_timeout=180)
client.get_transport().set_keepalive(5)
for metric in metric_names: for metric in metric_names:
try: try:
print(f"\t{metric}", end=" ", flush=True) print(f"\t{metric}", end=" ", flush=True)
command = "nice -n 19 influx -database cube-db -precision rfc3339 -execute \"SELECT * FROM RETENTION_CUBE_78w." + str(metric) + " WHERE time >= '" + str(start_date) + "' AND time < '" + str(end_date) + "'\" -format csv > /data/tmp/" + str(metric) + ".csv" command = "nice -n 19 influx -database cube-db -precision rfc3339 -execute \"SELECT * FROM RETENTION_CUBE_78w." + str(metric) + " WHERE time >= '" + str(start_date) + "' AND time < '" + str(end_date) + "'\" -format csv > /data/tmp/" + str(metric) + ".csv"
execute_sudo_ssh_command(ip_address, command) execute_sudo_ssh_command(ip_address, command, client)
print(f"✅", flush=True) print(f"✅", flush=True)
except Exception as e: except Exception as e:
print(f"❌", flush=True) print(f"❌", flush=True)
@@ -147,52 +165,31 @@ def main():
print(f"Skipping metric...", flush=True) print(f"Skipping metric...", flush=True)
continue continue
print(f"Downloading small datasets:", flush=True) try:
for metric in metric_names: command = "rm -rf /data/tmp/*.csv"
try: execute_sudo_ssh_command(ip_address, command, client)
print(f"\t{metric}", end=" ", flush=True) print(f"🗑️", flush=True)
absolute_remote_path = "/data/tmp/" + str(metric) + ".csv" except Exception as e:
relative_local_path = "./JUILLAUME_GOURET/" + str(i).zfill(2) + str(cube_id) + "/SMALL_DATASET/" + str(metric) + ".csv"
scp_get_file(ip_address, absolute_remote_path, relative_local_path)
print(f"✅", flush=True)
except Exception as e:
print(f"❌", flush=True) print(f"❌", flush=True)
print(f"Failed downloading {metric}", flush=True) print(f"Failed deleting CSV files", flush=True)
print(f"{e}", flush=True) print(f"{e}", flush=True)
print(f"Skipping metric...", flush=True)
continue continue
print(f"Getting medium datasets:", flush=True) # print(f"Downloading small datasets:", flush=True)
metric_names = ["SoC", "StrSoC"] # for metric in metric_names:
start_date = "2024-01-01" # try:
end_date = "2026-01-01" # print(f"\t{metric}", end=" ", flush=True)
for metric in metric_names: # activate_ssh(ip_address)
try: # absolute_remote_path = "/data/tmp/" + str(metric) + ".csv"
print(f"\t{metric}", end=" ", flush=True) # relative_local_path = "./JUILLAUME_GOURET_DANISH/" + str(cube_id) + "/" + str(metric) + ".csv"
command = "nice -n 19 influx -database cube-db -precision rfc3339 -execute \"SELECT * FROM RETENTION_CUBE_78w." + str(metric) + " WHERE time >= '" + str(start_date) + "' AND time < '" + str(end_date) + "'\" -format csv > /data/tmp/" + str(metric) + ".csv" # scp_get_file(ip_address, absolute_remote_path, relative_local_path)
execute_sudo_ssh_command(ip_address, command) # print(f"✅", flush=True)
print(f"✅", flush=True) # except Exception as e:
except Exception as e: # print(f"❌", flush=True)
print(f"❌", flush=True) # print(f"Failed downloading {metric}", flush=True)
print(f"Failed requesting {metric}:", flush=True) # print(f"{e}", flush=True)
print(f"{e}", flush=True) # print(f"Skipping metric...", flush=True)
print(f"Skipping metric...", flush=True) # continue
continue
print(f"Downloading medium datasets:", flush=True)
for metric in metric_names:
try:
print(f"\t{metric}", end=" ", flush=True)
absolute_remote_path = "/data/tmp/" + str(metric) + ".csv"
relative_local_path = "./JUILLAUME_GOURET/" + str(i).zfill(2) + str(cube_id) + "/MEDIUM_DATASET/" + str(metric) + ".csv"
scp_get_file(ip_address, absolute_remote_path, relative_local_path)
print(f"✅", flush=True)
except Exception as e:
print(f"❌", flush=True)
print(f"Failed downloading {metric}", flush=True)
print(f"{e}", flush=True)
print(f"Skipping metric...", flush=True)
continue
if __name__ == "__main__": if __name__ == "__main__":
main() main()