From 013d9c820fe516571f4a2acb3d1405f16befceb2 Mon Sep 17 00:00:00 2001 From: Quentin WEPHRE Date: Mon, 6 Oct 2025 08:04:06 +0200 Subject: [PATCH] [WiP] Upgrade for telemetry history --- .gitignore | 3 +- Python/cube_ssh_get_space.py | 123 +++++++++++++++++------------------ 2 files changed, 62 insertions(+), 64 deletions(-) diff --git a/.gitignore b/.gitignore index 1d8568d..429ece4 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ pyvenv.cfg Python/build/* Python/dist/* *.spec -dist/* \ No newline at end of file +dist/* +*.7z diff --git a/Python/cube_ssh_get_space.py b/Python/cube_ssh_get_space.py index 9d06bd0..8d58c27 100644 --- a/Python/cube_ssh_get_space.py +++ b/Python/cube_ssh_get_space.py @@ -21,8 +21,8 @@ def resource_path(relative_path): dotenv_path = resource_path('.env') load_dotenv(dotenv_path=dotenv_path) -ip_address_prefix = "10.84.171." # Myrtle subnet -ip_address_range = range(3, 5) # From 1 to 57 +ip_address_prefix = "10.84.165." # Danish subnet +ip_address_range = range(131, 132) # From 131 to 187 ENV_SSH = { "DEFAULT_CUBE_LINUX_ADMIN_USER": os.getenv("DEFAULT_CUBE_LINUX_ADMIN_USER"), @@ -32,37 +32,46 @@ ENV_SSH = { ssh_username = ENV_SSH["DEFAULT_CUBE_LINUX_ADMIN_USER"] ssh_password = ENV_SSH["DEFAULT_CUBE_LINUX_ADMIN_PASSWORD"] -def execute_ssh_command(ip, command): - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +def execute_ssh_command(ip, command, client): try: - client.connect(ip, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False) - stdin, stdout, stderr = client.exec_command(command) + stdin, stdout, stderr = client.exec_command(command, timeout=180) + exit_status = stdout.channel.recv_exit_status() + result = stdout.read().decode().lower().strip() + error = stderr.read().decode('utf-8') + + if exit_status == 0: + print(f"✅") + else: + print(f"❌") + print(f"{error}") + raise Exception(f"{str(error)}") return result except Exception as e: - print(f"SSH Error: {str(e)}", flush=True) + print(f"SSH error: {str(e)} --- {str(error)}", flush=True) raise finally: client.close() -def execute_sudo_ssh_command(ip, command): - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +def execute_sudo_ssh_command(ip, command, client): try: - client.connect(ip, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False) - client.get_transport().set_keepalive(10) - quoted_command = f"bash -c {shlex.quote(command)}" sudo_command = f"sudo -S -p '' {quoted_command}" - stdin, stdout, stderr = client.exec_command(sudo_command, timeout=60) + stdin, stdout, stderr = client.exec_command(sudo_command, timeout=180) time.sleep(3) stdin.write(ssh_password + '\n') stdin.flush() + exit_status = stdout.channel.recv_exit_status() - error = stderr.read().decode().strip() - if error: - raise Exception(f"sudo SSH command failed with error: {error}") + output = stdout.read().decode('utf-8') + error = stderr.read().decode('utf-8') + if exit_status == 0: + + print(f"{output}") + else: + print(f"❌") + print(f"{error}") + raise Exception("Error during SSH sudo command.") result = stdout.read().decode().lower().strip() return result @@ -90,9 +99,11 @@ def scp_get_file(ip, remote_path, local_path): username=ssh_username, password=ssh_password, allow_agent=False, - look_for_keys=False + look_for_keys=False, + timeout=180 ) + client.get_transport().set_keepalive(5) with SCPClient(client.get_transport()) as scp: scp.get(remote_path, local_path) @@ -131,14 +142,21 @@ def main(): continue print(f"Getting small datasets:", flush=True) - metric_names = [] #["ModTmpAvg", "ModTmpMin", "ModTmpMax", "StrModTmpAvg"] + metric_names = ["ModTmpAvg", "ModTmpMin", "ModTmpMax", "StrModTmpAvg", "NStrCon", "DCDischEnergy", "DCChEnergy", "SoC", "StrSoC"] start_date = "2024-01-01" end_date = "2026-01-01" + + activate_ssh(ip_address) + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect(ip_address, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False, timeout=180, banner_timeout=180) + client.get_transport().set_keepalive(5) + for metric in metric_names: try: print(f"\t{metric}", end=" ", flush=True) command = "nice -n 19 influx -database cube-db -precision rfc3339 -execute \"SELECT * FROM RETENTION_CUBE_78w." + str(metric) + " WHERE time >= '" + str(start_date) + "' AND time < '" + str(end_date) + "'\" -format csv > /data/tmp/" + str(metric) + ".csv" - execute_sudo_ssh_command(ip_address, command) + execute_sudo_ssh_command(ip_address, command, client) print(f"✅", flush=True) except Exception as e: print(f"❌", flush=True) @@ -147,52 +165,31 @@ def main(): print(f"Skipping metric...", flush=True) continue - print(f"Downloading small datasets:", flush=True) - for metric in metric_names: - try: - print(f"\t{metric}", end=" ", flush=True) - absolute_remote_path = "/data/tmp/" + str(metric) + ".csv" - relative_local_path = "./JUILLAUME_GOURET/" + str(i).zfill(2) + str(cube_id) + "/SMALL_DATASET/" + str(metric) + ".csv" - scp_get_file(ip_address, absolute_remote_path, relative_local_path) - print(f"✅", flush=True) - except Exception as e: + try: + command = "rm -rf /data/tmp/*.csv" + execute_sudo_ssh_command(ip_address, command, client) + print(f"🗑️", flush=True) + except Exception as e: print(f"❌", flush=True) - print(f"Failed downloading {metric}", flush=True) + print(f"Failed deleting CSV files", flush=True) print(f"{e}", flush=True) - print(f"Skipping metric...", flush=True) continue - print(f"Getting medium datasets:", flush=True) - metric_names = ["SoC", "StrSoC"] - start_date = "2024-01-01" - end_date = "2026-01-01" - for metric in metric_names: - try: - print(f"\t{metric}", end=" ", flush=True) - command = "nice -n 19 influx -database cube-db -precision rfc3339 -execute \"SELECT * FROM RETENTION_CUBE_78w." + str(metric) + " WHERE time >= '" + str(start_date) + "' AND time < '" + str(end_date) + "'\" -format csv > /data/tmp/" + str(metric) + ".csv" - execute_sudo_ssh_command(ip_address, command) - print(f"✅", flush=True) - except Exception as e: - print(f"❌", flush=True) - print(f"Failed requesting {metric}:", flush=True) - print(f"{e}", flush=True) - print(f"Skipping metric...", flush=True) - continue - - print(f"Downloading medium datasets:", flush=True) - for metric in metric_names: - try: - print(f"\t{metric}", end=" ", flush=True) - absolute_remote_path = "/data/tmp/" + str(metric) + ".csv" - relative_local_path = "./JUILLAUME_GOURET/" + str(i).zfill(2) + str(cube_id) + "/MEDIUM_DATASET/" + str(metric) + ".csv" - scp_get_file(ip_address, absolute_remote_path, relative_local_path) - print(f"✅", flush=True) - except Exception as e: - print(f"❌", flush=True) - print(f"Failed downloading {metric}", flush=True) - print(f"{e}", flush=True) - print(f"Skipping metric...", flush=True) - continue + # print(f"Downloading small datasets:", flush=True) + # for metric in metric_names: + # try: + # print(f"\t{metric}", end=" ", flush=True) + # activate_ssh(ip_address) + # absolute_remote_path = "/data/tmp/" + str(metric) + ".csv" + # relative_local_path = "./JUILLAUME_GOURET_DANISH/" + str(cube_id) + "/" + str(metric) + ".csv" + # scp_get_file(ip_address, absolute_remote_path, relative_local_path) + # print(f"✅", flush=True) + # except Exception as e: + # print(f"❌", flush=True) + # print(f"Failed downloading {metric}", flush=True) + # print(f"{e}", flush=True) + # print(f"Skipping metric...", flush=True) + # continue if __name__ == "__main__": main()