from paramiko import SSHClient, AutoAddPolicy import paramiko from cube_activate_ssh import activate_ssh from dotenv import load_dotenv import os import sys import shlex from scp import SCPClient import time def resource_path(relative_path): """ Get absolute path to resource, works for dev and for PyInstaller """ try: # PyInstaller creates a temp folder and stores path in _MEIPASS base_path = sys._MEIPASS except Exception: base_path = os.path.abspath(".") return os.path.join(base_path, relative_path) dotenv_path = resource_path('.env') load_dotenv(dotenv_path=dotenv_path) ip_address_prefix = "10.84.171." # Myrtle subnet ip_address_range = range(3, 5) # From 1 to 57 ENV_SSH = { "DEFAULT_CUBE_LINUX_ADMIN_USER": os.getenv("DEFAULT_CUBE_LINUX_ADMIN_USER"), "DEFAULT_CUBE_LINUX_ADMIN_PASSWORD": os.getenv("DEFAULT_CUBE_LINUX_ADMIN_PASSWORD") } ssh_username = ENV_SSH["DEFAULT_CUBE_LINUX_ADMIN_USER"] ssh_password = ENV_SSH["DEFAULT_CUBE_LINUX_ADMIN_PASSWORD"] def execute_ssh_command(ip, command): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: client.connect(ip, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False) stdin, stdout, stderr = client.exec_command(command) result = stdout.read().decode().lower().strip() return result except Exception as e: print(f"SSH Error: {str(e)}", flush=True) raise finally: client.close() def execute_sudo_ssh_command(ip, command): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: client.connect(ip, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False) client.get_transport().set_keepalive(10) quoted_command = f"bash -c {shlex.quote(command)}" sudo_command = f"sudo -S -p '' {quoted_command}" stdin, stdout, stderr = client.exec_command(sudo_command, timeout=60) time.sleep(3) stdin.write(ssh_password + '\n') stdin.flush() error = stderr.read().decode().strip() if error: raise Exception(f"sudo SSH command failed with error: {error}") result = stdout.read().decode().lower().strip() return result except Exception as e: raise finally: client.close() def scp_get_file(ip, remote_path, local_path): client = SSHClient() client.set_missing_host_key_policy(AutoAddPolicy()) local_path = os.path.expanduser(local_path) local_path = os.path.abspath(local_path) local_dir = os.path.dirname(local_path) if local_dir: os.makedirs(local_dir, exist_ok=True) try: client.connect( ip, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False ) with SCPClient(client.get_transport()) as scp: scp.get(remote_path, local_path) except Exception as e: raise finally: client.close() def main(): print(f"Starting...\n", flush=True) numbers = ip_address_range for i in numbers: ip_address = f"{ip_address_prefix}{i}" print(f"[{time.ctime(time.time())}] {str(i)} ({ip_address})", end=" ", flush=True) try: activate_ssh(ip_address) except Exception as e: print(f"SSH activation failed for {ip_address}:", flush=True) print(f"{e}", flush=True) print(f"Skipping CUBE...", flush=True) continue cube_id = "NA" try: cube_id = execute_ssh_command(ip_address, "hostname") print(f"{cube_id} ✅", flush=True) except Exception as e: print(f"cube-xxxxx ❌", flush=True) print(f"Error getting hostname for {ip_address}:", flush=True) print(f"{e}", flush=True) print(f"Skipping CUBE...", flush=True) continue print(f"Getting small datasets:", flush=True) metric_names = [] #["ModTmpAvg", "ModTmpMin", "ModTmpMax", "StrModTmpAvg"] start_date = "2024-01-01" end_date = "2026-01-01" for metric in metric_names: try: print(f"\t{metric}", end=" ", flush=True) command = "nice -n 19 influx -database cube-db -precision rfc3339 -execute \"SELECT * FROM RETENTION_CUBE_78w." + str(metric) + " WHERE time >= '" + str(start_date) + "' AND time < '" + str(end_date) + "'\" -format csv > /data/tmp/" + str(metric) + ".csv" execute_sudo_ssh_command(ip_address, command) print(f"✅", flush=True) except Exception as e: print(f"❌", flush=True) print(f"Failed requesting {metric}:", flush=True) print(f"{e}", flush=True) print(f"Skipping metric...", flush=True) continue print(f"Downloading small datasets:", flush=True) for metric in metric_names: try: print(f"\t{metric}", end=" ", flush=True) absolute_remote_path = "/data/tmp/" + str(metric) + ".csv" relative_local_path = "./JUILLAUME_GOURET/" + str(i).zfill(2) + str(cube_id) + "/SMALL_DATASET/" + str(metric) + ".csv" scp_get_file(ip_address, absolute_remote_path, relative_local_path) print(f"✅", flush=True) except Exception as e: print(f"❌", flush=True) print(f"Failed downloading {metric}", flush=True) print(f"{e}", flush=True) print(f"Skipping metric...", flush=True) continue print(f"Getting medium datasets:", flush=True) metric_names = ["SoC", "StrSoC"] start_date = "2024-01-01" end_date = "2026-01-01" for metric in metric_names: try: print(f"\t{metric}", end=" ", flush=True) command = "nice -n 19 influx -database cube-db -precision rfc3339 -execute \"SELECT * FROM RETENTION_CUBE_78w." + str(metric) + " WHERE time >= '" + str(start_date) + "' AND time < '" + str(end_date) + "'\" -format csv > /data/tmp/" + str(metric) + ".csv" execute_sudo_ssh_command(ip_address, command) print(f"✅", flush=True) except Exception as e: print(f"❌", flush=True) print(f"Failed requesting {metric}:", flush=True) print(f"{e}", flush=True) print(f"Skipping metric...", flush=True) continue print(f"Downloading medium datasets:", flush=True) for metric in metric_names: try: print(f"\t{metric}", end=" ", flush=True) absolute_remote_path = "/data/tmp/" + str(metric) + ".csv" relative_local_path = "./JUILLAUME_GOURET/" + str(i).zfill(2) + str(cube_id) + "/MEDIUM_DATASET/" + str(metric) + ".csv" scp_get_file(ip_address, absolute_remote_path, relative_local_path) print(f"✅", flush=True) except Exception as e: print(f"❌", flush=True) print(f"Failed downloading {metric}", flush=True) print(f"{e}", flush=True) print(f"Skipping metric...", flush=True) continue if __name__ == "__main__": main()